From bc2937a52417735bc29111ae9ace12c5e58ec2bc Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 13:39:14 +0100 Subject: [PATCH 01/30] =?UTF-8?q?docs:=20add=20spec=2034=20=E2=80=94=20Pol?= =?UTF-8?q?ars=20engine=20conversion=20design?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design for replacing the DuckDB engine with a pure-Rust Polars engine while preserving the existing CLI and SQL interface. Motivated by a real silent-hang failure in the duckdb crate on a malformed CSV field. Co-Authored-By: Claude Opus 4.7 --- docs/specs/34-polars-engine.md | 161 +++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 docs/specs/34-polars-engine.md diff --git a/docs/specs/34-polars-engine.md b/docs/specs/34-polars-engine.md new file mode 100644 index 0000000..2e298cb --- /dev/null +++ b/docs/specs/34-polars-engine.md @@ -0,0 +1,161 @@ +# Polars Engine (replaces DuckDB Engine) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:writing-plans to turn this design into an implementation plan, then superpowers:subagent-driven-development or superpowers:executing-plans to implement it. + +**Goal:** Replace the in-memory DuckDB engine (spec 03) with a Polars-based engine that powers the same file reading, SQL execution, and data accumulation — while preserving dtoo's existing CLI and SQL interface. + +**Supersedes:** `docs/specs/03-duckdb-engine.md`. + +**Tech Stack:** Rust, `polars` crate (pure Rust), `polars-sql` (`SQLContext`). + +**Tier:** Open Source + +--- + +## Motivation + +This is not a speculative rewrite. The driving problem is concrete and was experienced in real use: + +> While trialling dtoo at work, the `duckdb` crate **silently hung** on a malformed field in a test CSV and never propagated an error back to Rust. Half a day was lost to suspected linking issues before the DuckDB CLI revealed it was a bad CSV field. A tool that can hang indefinitely with no error is not shippable to users. + +The fix is an engine whose failures are ordinary Rust `Result`s. Polars is pure Rust: read errors, malformed-row errors, and unsupported-SQL errors all return `PolarsError` with useful messages — **no opaque C++ hangs**. The secondary benefit is a pure-Rust build with no bundled C++ toolchain. + +## Scope + +**In scope (this conversion):** the open-source command surface that exists today — `query`, `convert`, `profile`, `inspect`, `fingerprint`, plus crypto profiles, masking, lineage, reference tables, schema handling, output writing, manifests, fingerprinting. + +**Deferred (follow-up specs):** cloud storage (S3/GCS/Azure). Polars supports it, but the `aws`/`gcp`/`azure` features pull in `object_store` + `tokio` + a TLS stack — a heavy async tree that works against the lean pure-Rust goal. We ship the local-file core first, then add cloud back once the engine swap is proven. + +**Out of scope:** the Pro/Enterprise features that are spec-only and unimplemented (assertions, sinks, partitioning, pipelines, audit log, webhooks, parallel). Their specs are unaffected by this change and will be re-evaluated when built. + +## Architectural Shift + +Today, **every** stage mutates a hidden `temp_results` table by issuing SQL strings through `DuckDbEngine` — masking, lineage, limit, crypto, profiling, and schema evolution all generate SQL (including DuckDB-only functions like `hmac()`). The data is never a value; it is a side effect inside the connection. + +The new model carries the data **as a value** — a Polars `LazyFrame` — through the pipeline. Only the *user-facing* SQL stays SQL. + +``` +files ── scan_csv / scan_parquet / scan_ndjson (glob native) ──▶ Vec + └─ concat_lf_diagonal(..) // == today's "UNION ALL BY NAME" schema evolution + │ + USER SQL via SQLContext: + register("_", lf); register each --ref; execute(--where / --filter-sql / --post-sql) + │ + INTERNAL stages as native Polars expressions (NO generated SQL): + masking → existing HMAC-SHA256 Rust code, applied via with_column + lineage → with_columns(batch_id / record_id / batch_timestamp / batch_hash / origin_file) + limit → .limit(n) + crypto → existing AES/ECB Rust code applied via expressions + │ + .collect() ──▶ DataFrame ──▶ CsvWriter / ParquetWriter / JsonWriter (file or stdout) +``` + +This is cleaner than the current design: internal transforms become honest, typed Rust operations instead of generated SQL, and they no longer depend on DuckDB-only SQL functions. + +## Engine API + +Replace `DuckDbEngine` with `PolarsEngine`. Rather than `execute(sql)` against a mutable table, the engine exposes value-passing operations: + +- `scan(path, format) -> Result` — readers for Parquet / CSV (custom delimiter) / NDJSON / Excel (sheet by name). +- `concat_by_name(frames) -> Result` — union-by-name with type coercion (`concat_lf_diagonal`). +- `run_sql(lf, refs, sql) -> Result` — register `_` and refs in a `SQLContext`, execute, return the resulting `LazyFrame`. Surfaces unsupported-SQL as a clean error. +- `schema(lf) -> Result` — `collect_schema()` without materializing (for dry-run, schema mode, profiling). +- `collect(lf) -> Result` and `row_count(lf) -> Result`. +- `write(df, dest, format, header, delimiter, compression) -> Result<()>` — writers, to a file path or stdout, wrapping the writer in `flate2`/`zstd` for CSV/NDJSON compression. + +The `_` magic table and reserved names (`_`, `temp_results`) are preserved exactly as today. + +## Stage Mapping (DuckDB → Polars) + +| Stage | Today (DuckDB SQL) | New (Polars) | +|-------|--------------------|--------------| +| Read file as `_` | `CREATE VIEW _ AS read_csv(...)` | `scan_csv(...)`, registered as `_` in `SQLContext` | +| Reference tables | `CREATE TABLE r AS read_*` | `scan_*` → `ctx.register("r", lf)` | +| Schema evolution | `INSERT ... BY NAME` / `UNION ALL BY NAME` | `concat_lf_diagonal` | +| `--where` | `SELECT * FROM _ WHERE ..` | same SQL via `SQLContext` | +| `--filter-sql` / `--post-sql` | raw SQL on `_` | same SQL via `SQLContext` | +| Explicit `--schema` | `CREATE TABLE (...)` + coerce | cast columns to declared dtypes; missing → null, extra → dropped | +| Masking | SQL `hmac(...)` UPDATE | existing Rust HMAC, applied via `with_column` | +| Lineage | SQL column adds | `with_columns` expressions | +| Crypto | SQL transforms | existing Rust AES/ECB via expressions | +| `--limit` | `LIMIT n` rewrite | `.limit(n)` | +| Profiling | DuckDB aggregates | Polars aggregate expressions (see below) | +| Export | `COPY ... TO` | `CsvWriter`/`ParquetWriter`/`JsonWriter` | + +## SQL Surface: What Users Gain and Lose + +`SQLContext` covers the dtoo SQL use cases: `SELECT`, `WHERE`, `GROUP BY` + aggregates, `INNER`/`LEFT`/`RIGHT`/`FULL`/`CROSS JOIN`, `ORDER BY`, `LIMIT`, CTEs, `UNION`/`UNION ALL`, subqueries, `CASE`, and common string/date functions. Unsupported SQL returns a clear `Result` error — never a hang. + +**Known limitations to document in the user guide:** + +1. **Window functions** (`OVER (PARTITION BY .. ORDER BY ..)`) have known correctness bugs in Polars SQL. Document as unsupported; do not silently emit wrong results. +2. **Narrower function library** than DuckDB — some exotic date/regex/string functions are absent. They fail with a clear "unsupported function" error. +3. **CSV/NDJSON output compression** is not native to Polars — implemented by wrapping the writer in `flate2` (gzip) / `zstd`. Parquet codecs remain built in. + +## Dependencies + +**Add:** +```toml +polars = { version = "1", features = ["lazy", "sql", "csv", "parquet", "json", "excel"] } +# additional dtype-* features (e.g. decimal, datetime) selected during implementation as the +# porting of schema/profiling reveals which are required +flate2 = "1" # gzip output for CSV/NDJSON (Polars has no native text-output compression) +zstd = "0.13" # zstd output for CSV/NDJSON +``` +`excel` pulls in the pure-Rust `calamine` backend (no C/C++). Cloud features are intentionally **not** enabled in this phase. + +**Remove:** +```toml +duckdb = { version = "1", features = ["bundled"] } # eliminates bundled C++ toolchain +glob = "0.3" # Polars scans globs natively +``` + +`flate2`/`zstd` are justified by a capability Polars lacks (text-output compression that dtoo already advertises via `--compress`); they are small, pure-Rust, and widely used. + +## Cloud Deferral Behaviour + +CLI flags (`--s3-region`, `--s3-profile`, `--gcs-project`, `--azure-account`) remain parsed for forward compatibility. A cloud path (`s3://`, `gs://`, `az://`) in any input/ref/output position returns an explicit error: + +> `cloud storage (s3://…) is not supported in this build yet` + +This keeps the CLI surface stable and avoids the previous silent-failure class. A later spec re-enables cloud behind the Polars `cloud`/`aws`/`gcp`/`azure` features. + +## Profiling with Polars + +Per-column statistics map to Polars aggregate expressions: `null_count`, `n_unique` (with `approx_n_unique` for large inputs), `min`/`max`/`mean`/`std`/`median`, `quantile(0.25)`/`quantile(0.75)`, string `len_chars` stats, and top-N via `value_counts` + sort + head. There is no single `describe()` on `LazyFrame`; the profiler builds the aggregation list explicitly per dtype. HTML/JSON/CSV report formats are unchanged. + +## Migration Strategy + +Convert in slices, keeping `cargo test` green at every step. The existing inline pipeline tests and `tests/` integration tests assert *behaviour*, not engine internals, so they are the regression safety net. + +1. **Readers + simple commands:** `scan_*` + glob; convert `convert`, `inspect`, `fingerprint`. +2. **Query core:** scan → `concat_lf_diagonal` → `SQLContext` (`--where`/`--filter-sql`/`--post-sql`/refs) → write. Includes schema mode. +3. **Internal expression stages:** masking, lineage, limit as native Polars expressions. +4. **Profiler:** port aggregate queries to Polars expressions. +5. **Crypto:** port decrypt/encrypt stages to expression-based application. +6. **Cleanup:** delete DuckDB-specific code (`sql_utils` escaping helpers no longer needed for internal SQL), update `DESIGN.md` and the user guide (SQL limitations, cloud deferral). + +## Error Handling + +- File read / malformed data → `PolarsError` surfaced with the file path. **Never hangs** (the originating motivation). +- Unsupported SQL → clear error naming the construct, returned as `Result`. +- Schema coercion failure under explicit `--schema` → error naming the column and target dtype. +- Cloud path → explicit "not supported in this build" error (see above). + +## Testing + +- Keep all existing behavioural tests passing; they define the conversion's success. +- Add tests for the new honest-error paths: malformed CSV field returns an error (not a hang), unsupported window-function SQL returns a clear error. +- Add a union-by-name test across files with differing/overlapping columns (`concat_lf_diagonal`). +- Verify CSV/NDJSON gzip and zstd output round-trip. + +--- + +## Appendix: Deferred Feature Ideas (separate specs, after the conversion) + +Captured here so they are not lost; **not** part of this conversion. Each would be its own spec/PR, and several need a "does this belong in core?" check per `CLAUDE.md` before pursuing: + +- **`dtoo dbt-tests`** — profile a dataset and emit a proposed dbt `schema.yml` with tests (`not_null`, `unique`, `accepted_values`, range/`between`, `relationships`, freshness) derived from profiling stats. Strong fit: dtoo already profiles. The headline idea from the original request. +- **`dtoo dbt-model`** — scaffold a dbt staging model (`.sql` + `.yml`) from a source file's schema: snake_case renames, type casts, a stub `sources.yml`. +- **BigQuery schema emission** — from profiling/schema, emit a BigQuery DDL or `bq load --schema` JSON, with a Polars→BQ type mapping. Emits artifacts/commands rather than adding a network client, keeping the build lean. +- **Polars code generation** — emit an equivalent Polars (Python or Rust) snippet for a given dtoo query, as a learning/bridge aid. Niche; lowest priority. From 7e308658ec9cd79ccbdcdae39376e14759dd7bfd Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 13:48:34 +0100 Subject: [PATCH 02/30] docs: strengthen spec 34 motivation with runtime-extension constraint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DuckDB downloads httpfs/excel/azure extensions at runtime, which fails in no-binary-download environments — a second concrete argument for the pure-Rust Polars engine. Co-Authored-By: Claude Opus 4.7 --- docs/specs/34-polars-engine.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/specs/34-polars-engine.md b/docs/specs/34-polars-engine.md index 2e298cb..bb34a9c 100644 --- a/docs/specs/34-polars-engine.md +++ b/docs/specs/34-polars-engine.md @@ -20,6 +20,13 @@ This is not a speculative rewrite. The driving problem is concrete and was exper The fix is an engine whose failures are ordinary Rust `Result`s. Polars is pure Rust: read errors, malformed-row errors, and unsupported-SQL errors all return `PolarsError` with useful messages — **no opaque C++ hangs**. The secondary benefit is a pure-Rust build with no bundled C++ toolchain. +A second, environment-specific reason: dtoo's target deployment is a locked-down +corporate setting where **binaries cannot be downloaded** — everything builds from +source. DuckDB loads its `httpfs`, `excel`, and `azure` extensions by downloading +them at runtime (`INSTALL …`), so the Excel and cloud paths would fail there +regardless of the hang. Polars compiles Excel support in (pure-Rust `calamine`) and +needs no runtime extension fetch. + ## Scope **In scope (this conversion):** the open-source command surface that exists today — `query`, `convert`, `profile`, `inspect`, `fingerprint`, plus crypto profiles, masking, lineage, reference tables, schema handling, output writing, manifests, fingerprinting. From c48f6db4a449f4a2b90877b3fcddd2764abe2152 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 15:30:16 +0100 Subject: [PATCH 03/30] docs: add Phase 1 implementation plan for Polars engine core Bite-sized TDD plan to build and unit-test PolarsEngine (readers, SQL, schema, writers, compression, cloud guard) alongside the DuckDB engine, keeping the build green. Cutover is Phase 2. Co-Authored-By: Claude Opus 4.7 --- .../plans/2026-06-06-polars-engine-core.md | 1056 +++++++++++++++++ 1 file changed, 1056 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-06-polars-engine-core.md diff --git a/docs/superpowers/plans/2026-06-06-polars-engine-core.md b/docs/superpowers/plans/2026-06-06-polars-engine-core.md new file mode 100644 index 0000000..f3222f5 --- /dev/null +++ b/docs/superpowers/plans/2026-06-06-polars-engine-core.md @@ -0,0 +1,1056 @@ +# Polars Engine Core — Implementation Plan (Phase 1 of 2) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build and unit-test a new pure-Rust `PolarsEngine` (file readers, SQL execution, schema introspection, writers, compression, cloud guard) that lives *alongside* the existing DuckDB engine without touching the running pipeline. + +**Architecture:** A new `src/polars_engine.rs` module exposing a `PolarsEngine` that carries data as a Polars `LazyFrame`/`DataFrame` value. It is added to the binary behind `#[allow(dead_code)]` so the crate keeps compiling and all existing tests keep passing while we build and verify each operation in isolation. Phase 2 (a separate plan) rewires the pipeline onto this engine and deletes DuckDB. + +**Tech Stack:** Rust, `polars` (lazy, sql, csv, parquet, json), `calamine` (Excel), `flate2` (gzip), `zstd` (zstd). Reuses existing enums `InputFormat` / `ExportFormat` / `CompressionCodec` from `src/engine.rs` and the `DtooError` type from `src/error.rs`. + +**Spec:** `docs/specs/34-polars-engine.md`. + +**Why two phases:** the whole codebase depends on `DuckDbEngine` and will not compile half-converted. Building `PolarsEngine` in parallel keeps the build green and lets us prove every primitive with unit tests before the risky cutover. Phase 2's exact rewiring code targets the *realized* API from this phase, so it is written after this phase compiles — not speculatively. + +**Reference — error mapping used throughout this module** (variants confirmed in `src/error.rs`): +- File read / parse failures → `DtooError::FileProcess { path, message }` (renders `Failed to process {path}: {message}` — the clear, non-hanging error the conversion exists to deliver). +- SQL execution failures → `DtooError::Sql { context, sql, source }`. +- Write failures → `DtooError::Output { message }`. +- Schema introspection failures → `DtooError::Schema { message }`. +- Cloud path rejection (deferred feature) → `DtooError::Config { message }`. + +--- + +### Task 1: Add dependencies and create the module skeleton + +**Files:** +- Modify: `Cargo.toml` (dependencies) +- Create: `src/polars_engine.rs` +- Modify: `src/main.rs` (add `mod polars_engine;`) + +- [ ] **Step 1: Add the new dependencies** + +Use `cargo add` so the latest compatible versions are pinned automatically (do **not** hand-write version numbers; do **not** remove `duckdb` yet — both engines coexist during Phase 1). + +Run: +```bash +cargo add polars --features lazy,sql,csv,parquet,json,strings,dtype-full +cargo add calamine +cargo add flate2 +cargo add zstd +``` +Expected: `Cargo.toml` gains `polars`, `calamine`, `flate2`, `zstd`. If the `dtype-full` or `strings` feature is rejected by the resolver, drop it from the command and re-run — the remaining features are the required ones. + +- [ ] **Step 2: Create the module skeleton with the error helpers** + +Create `src/polars_engine.rs`: +```rust +//! Pure-Rust data engine built on Polars. Phase 1: built alongside the DuckDB +//! engine and not yet wired into the pipeline. See docs/specs/34-polars-engine.md. +#![allow(dead_code)] // removed in Phase 2 when the pipeline is rewired onto this engine + +use std::path::Path; + +use polars::prelude::*; + +use crate::engine::{CompressionCodec, ExportFormat, InputFormat}; +use crate::error::DtooError; + +/// Stateless handle for Polars-backed data operations. +pub struct PolarsEngine; + +impl PolarsEngine { + /// Construct a new engine handle. + pub fn new() -> Self { + Self + } +} + +fn read_err(path: &str, source: impl std::fmt::Display) -> DtooError { + DtooError::FileProcess { + path: path.to_string(), + message: source.to_string(), + } +} + +fn write_err(source: impl std::fmt::Display) -> DtooError { + DtooError::Output { + message: source.to_string(), + } +} + +fn sql_err(sql: &str, source: PolarsError) -> DtooError { + DtooError::Sql { + context: "polars-sql".to_string(), + sql: sql.to_string(), + source: Box::new(std::io::Error::other(source.to_string())), + } +} +``` + +- [ ] **Step 3: Register the module** + +In `src/main.rs`, add the module declaration alongside the other `mod` lines (keep alphabetical order if the file uses it): +```rust +mod polars_engine; +``` + +- [ ] **Step 4: Verify it compiles with no warnings** + +Run: `cargo build && cargo clippy --all-targets -- -D warnings` +Expected: builds cleanly; no warnings (the `#![allow(dead_code)]` suppresses unused-code warnings for the new module). + +- [ ] **Step 5: Commit** + +```bash +git add Cargo.toml Cargo.lock src/polars_engine.rs src/main.rs +git commit -m "feat: scaffold PolarsEngine module alongside DuckDB engine" +``` + +--- + +### Task 2: Scan CSV + collect + row_count + +**Files:** +- Modify: `src/polars_engine.rs` +- Test: inline `#[cfg(test)]` module in `src/polars_engine.rs` + +- [ ] **Step 1: Write the failing test** + +Add at the bottom of `src/polars_engine.rs`: +```rust +#[cfg(test)] +mod tests { + use super::*; + + fn tmp(name: &str, ext: &str) -> std::path::PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("dtoo-pe-{name}-{nanos}.{ext}")) + } + + #[test] + fn scan_csv_reads_rows_and_columns() { + let path = tmp("csv", "csv"); + std::fs::write(&path, "id,name\n1,alice\n2,bob\n").unwrap(); + let engine = PolarsEngine::new(); + + let lf = engine + .scan(path.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(); + let count = engine.row_count(lf.clone()).unwrap(); + let df = engine.collect(lf).unwrap(); + + assert_eq!(count, 2); + assert_eq!(df.get_column_names_str(), vec!["id", "name"]); + let _ = std::fs::remove_file(path); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --lib polars_engine::tests::scan_csv_reads_rows_and_columns` +Expected: FAIL — `scan`, `row_count`, `collect` not defined. + +- [ ] **Step 3: Implement scan (CSV branch), collect, row_count** + +Add these methods inside `impl PolarsEngine`: +```rust + /// Lazily scan an input file by format. Glob patterns are supported by the + /// underlying Polars scanners for Parquet/CSV/NDJSON. + pub fn scan(&self, path: &str, format: &InputFormat) -> Result { + reject_cloud(path)?; + match format { + InputFormat::Csv { delimiter } => LazyCsvReader::new(path) + .with_separator(*delimiter as u8) + .with_has_header(true) + .finish() + .map_err(|e| read_err(path, e)), + _ => Err(read_err(path, "unsupported format (implemented in a later task)")), + } + } + + /// Materialize a LazyFrame into a DataFrame. + pub fn collect(&self, lf: LazyFrame) -> Result { + lf.collect().map_err(|e| read_err("(query)", e)) + } + + /// Count rows without retaining the materialized frame. + pub fn row_count(&self, lf: LazyFrame) -> Result { + Ok(self.collect(lf)?.height()) + } +``` + +Also add the cloud guard near the bottom of the file (full behaviour tested in Task 13): +```rust +fn reject_cloud(path: &str) -> Result<(), DtooError> { + if crate::path_utils::is_cloud_path(path) { + return Err(DtooError::Config { + message: format!("cloud storage ({path}) is not supported in this build yet"), + }); + } + Ok(()) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::scan_csv_reads_rows_and_columns` +Expected: PASS. + +> Note: if `get_column_names_str` is not the method name in the pinned Polars version, use `df.get_column_names()` and compare against `&["id", "name"]`. The compile error will tell you; pick the one that exists. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: PolarsEngine CSV scan, collect, row_count" +``` + +--- + +### Task 3: Scan NDJSON + +**Files:** +- Modify: `src/polars_engine.rs` + +- [ ] **Step 1: Write the failing test** + +Add to the `tests` module: +```rust + #[test] + fn scan_ndjson_reads_rows() { + let path = tmp("ndjson", "ndjson"); + std::fs::write(&path, "{\"id\":1,\"name\":\"alice\"}\n{\"id\":2,\"name\":\"bob\"}\n").unwrap(); + let engine = PolarsEngine::new(); + + let lf = engine.scan(path.to_str().unwrap(), &InputFormat::Ndjson).unwrap(); + assert_eq!(engine.row_count(lf).unwrap(), 2); + let _ = std::fs::remove_file(path); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --lib polars_engine::tests::scan_ndjson_reads_rows` +Expected: FAIL — NDJSON branch returns the "unsupported format" error. + +- [ ] **Step 3: Implement the NDJSON branch** + +In `scan`, replace the `InputFormat::Ndjson` path inside the `match` (add a new arm before the `_ =>` catch-all): +```rust + InputFormat::Ndjson => LazyJsonLineReader::new(path) + .finish() + .map_err(|e| read_err(path, e)), +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::scan_ndjson_reads_rows` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: PolarsEngine NDJSON scan" +``` + +--- + +### Task 4: Write CSV (to file and to stdout sink) + +**Files:** +- Modify: `src/polars_engine.rs` + +- [ ] **Step 1: Write the failing test** + +Add to the `tests` module: +```rust + #[test] + fn write_csv_roundtrips() { + let src = tmp("wcsv-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("wcsv-dst", "csv"); + let engine = PolarsEngine::new(); + + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine + .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', None) + .unwrap(); + + let written = std::fs::read_to_string(&dst).unwrap(); + assert!(written.contains("id,name")); + assert!(written.contains("1,alice")); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --lib polars_engine::tests::write_csv_roundtrips` +Expected: FAIL — `write` not defined. + +- [ ] **Step 3: Implement write (CSV branch) with a stdout-or-file sink** + +Add a sink helper and the `write` method: +```rust + /// Write a DataFrame to a file path or, when `dest` is None, to stdout. + pub fn write( + &self, + mut df: DataFrame, + dest: Option<&Path>, + format: ExportFormat, + header: bool, + delimiter: char, + compression: Option, + ) -> Result<(), DtooError> { + let sink = open_sink(dest)?; + match format { + ExportFormat::Csv => { + let writer = wrap_compression(sink, compression); + CsvWriter::new(writer) + .include_header(header) + .with_separator(delimiter as u8) + .finish(&mut df) + .map_err(write_err) + } + _ => Err(write_err("unsupported export format (implemented in a later task)")), + } + } +``` + +Add the sink + compression helpers near the bottom of the file: +```rust +fn open_sink(dest: Option<&Path>) -> Result, DtooError> { + match dest { + Some(path) => { + let file = std::fs::File::create(path) + .map_err(|e| write_err(format!("{}: {e}", path.display())))?; + Ok(Box::new(file)) + } + None => Ok(Box::new(std::io::stdout())), + } +} + +fn wrap_compression( + sink: Box, + compression: Option, +) -> Box { + match compression { + None => sink, + Some(CompressionCodec::Gzip) => { + Box::new(flate2::write::GzEncoder::new(sink, flate2::Compression::default())) + } + Some(CompressionCodec::Zstd) => { + Box::new(zstd::stream::write::AutoFinishEncoder::new( + zstd::stream::write::Encoder::new(sink, 0) + .expect("zstd encoder init") + .auto_finish(), + )) + } + } +} +``` + +> Note on zstd: the exact constructor for an auto-finishing encoder may differ slightly by crate version. If the above does not compile, use `zstd::stream::write::Encoder::new(sink, 0)?.auto_finish()` and box that directly — the goal is a `Box` that flushes the zstd frame on drop. The gzip path via `flate2::write::GzEncoder` finishes on drop. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::write_csv_roundtrips` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: PolarsEngine CSV writer with file/stdout sink" +``` + +--- + +### Task 5: Scan + write Parquet (roundtrip) + +**Files:** +- Modify: `src/polars_engine.rs` + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn parquet_roundtrips() { + let src = tmp("pq-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n2,bob\n").unwrap(); + let pq = tmp("pq-mid", "parquet"); + let engine = PolarsEngine::new(); + + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine.write(df, Some(pq.as_path()), ExportFormat::Parquet, true, ',', None).unwrap(); + + let back = engine.scan(pq.to_str().unwrap(), &InputFormat::Parquet).unwrap(); + assert_eq!(engine.row_count(back).unwrap(), 2); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(pq); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --lib polars_engine::tests::parquet_roundtrips` +Expected: FAIL — Parquet branch missing in both `scan` and `write`. + +- [ ] **Step 3: Implement Parquet scan + write** + +In `scan`, add an arm before the `_ =>` catch-all: +```rust + InputFormat::Parquet => LazyFrame::scan_parquet(path, ScanArgsParquet::default()) + .map_err(|e| read_err(path, e)), +``` + +In `write`, add an arm before the `_ =>` catch-all: +```rust + ExportFormat::Parquet => { + let sink = open_sink(dest)?; + let codec = match compression { + Some(CompressionCodec::Gzip) => ParquetCompression::Gzip(None), + Some(CompressionCodec::Zstd) => ParquetCompression::Zstd(None), + None => ParquetCompression::default(), + }; + ParquetWriter::new(sink) + .with_compression(codec) + .finish(&mut df) + .map(|_| ()) + .map_err(write_err) + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::parquet_roundtrips` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: PolarsEngine Parquet scan and writer" +``` + +--- + +### Task 6: Write NDJSON (roundtrip) + +**Files:** +- Modify: `src/polars_engine.rs` + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn ndjson_write_roundtrips() { + let src = tmp("wnd-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("wnd-dst", "ndjson"); + let engine = PolarsEngine::new(); + + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine.write(df, Some(dst.as_path()), ExportFormat::Ndjson, true, ',', None).unwrap(); + + let back = engine.scan(dst.to_str().unwrap(), &InputFormat::Ndjson).unwrap(); + assert_eq!(engine.row_count(back).unwrap(), 1); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --lib polars_engine::tests::ndjson_write_roundtrips` +Expected: FAIL — NDJSON export branch missing. + +- [ ] **Step 3: Implement the NDJSON write branch** + +In `write`, add an arm before the `_ =>` catch-all: +```rust + ExportFormat::Ndjson => { + let writer = wrap_compression(open_sink(dest)?, compression); + JsonWriter::new(writer) + .with_json_format(JsonFormat::JsonLines) + .finish(&mut df) + .map_err(write_err) + } +``` +Now remove the `_ =>` catch-all from `write` (all three `ExportFormat` variants are handled). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::ndjson_write_roundtrips` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: PolarsEngine NDJSON writer" +``` + +--- + +### Task 7: CSV/NDJSON output compression (gzip + zstd) + +**Files:** +- Modify: `src/polars_engine.rs` + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn csv_gzip_output_is_decompressible() { + let src = tmp("gz-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("gz-dst", "csv.gz"); + let engine = PolarsEngine::new(); + + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine + .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', Some(CompressionCodec::Gzip)) + .unwrap(); + + let bytes = std::fs::read(&dst).unwrap(); + let mut decoder = flate2::read::GzDecoder::new(&bytes[..]); + let mut text = String::new(); + std::io::Read::read_to_string(&mut decoder, &mut text).unwrap(); + assert!(text.contains("1,alice")); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } +``` + +- [ ] **Step 2: Run test to verify it fails OR passes** + +Run: `cargo test --lib polars_engine::tests::csv_gzip_output_is_decompressible` +Expected: This exercises the `wrap_compression` path already written in Task 4. If it PASSES immediately, the compression wiring is correct — keep the test as a regression guard and proceed to Step 5. If it FAILS (e.g. the gzip stream isn't flushed because the encoder was not finished on drop), continue to Step 3. + +- [ ] **Step 3: Fix flush-on-drop if needed** + +If the gzip output was truncated, the `Box` was dropped before `finish()`. Ensure `write` drops the wrapped writer before returning by scoping it: +```rust + ExportFormat::Csv => { + { + let writer = wrap_compression(open_sink(dest)?, compression); + CsvWriter::new(writer) + .include_header(header) + .with_separator(delimiter as u8) + .finish(&mut df) + .map_err(write_err)?; + } // writer dropped here -> gzip/zstd frame finalized + Ok(()) + } +``` +Apply the same scoping pattern to the NDJSON branch. + +- [ ] **Step 4: Add and run the zstd variant test** + +```rust + #[test] + fn csv_zstd_output_is_decompressible() { + let src = tmp("zs-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("zs-dst", "csv.zst"); + let engine = PolarsEngine::new(); + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine + .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', Some(CompressionCodec::Zstd)) + .unwrap(); + + let bytes = std::fs::read(&dst).unwrap(); + let text = String::from_utf8(zstd::stream::decode_all(&bytes[..]).unwrap()).unwrap(); + assert!(text.contains("1,alice")); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } +``` +Run: `cargo test --lib polars_engine::tests::csv_zstd_output_is_decompressible` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: gzip/zstd compression for CSV/NDJSON output" +``` + +--- + +### Task 8: Scan Excel via calamine + +**Files:** +- Modify: `src/polars_engine.rs` + +> **Design note:** Polars Rust has no lazy Excel scanner, so Excel is read eagerly with `calamine` and converted to an all-`String` DataFrame (every cell stringified). Downstream SQL/casts and explicit `--schema` refine types — this matches treating spreadsheets as untyped text and is intentionally simple for Phase 1. Type inference parity with DuckDB's `read_xlsx` is a documented follow-up, not a Phase-1 goal. This refines the spec's dependency note (`calamine` is a direct dependency, not pulled via a Polars feature). + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn scan_excel_reads_existing_fixture() { + // Repo fixture from testdata/. Default (first) sheet. + let engine = PolarsEngine::new(); + let lf = engine + .scan("testdata/xlsxs/trips.xlsx", &InputFormat::Excel { sheet: None }) + .unwrap(); + let df = engine.collect(lf).unwrap(); + assert!(df.height() > 0); + assert!(df.width() > 0); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --lib polars_engine::tests::scan_excel_reads_existing_fixture` +Expected: FAIL — Excel branch returns the catch-all error. + +- [ ] **Step 3: Implement the Excel branch** + +In `scan`, add an arm before the `_ =>` catch-all: +```rust + InputFormat::Excel { sheet } => read_excel(path, sheet.as_deref()), +``` + +Add the `read_excel` free function near the bottom of the file: +```rust +fn read_excel(path: &str, sheet: Option<&str>) -> Result { + use calamine::{open_workbook_auto, Data, Reader}; + + let mut workbook = open_workbook_auto(path).map_err(|e| read_err(path, e))?; + let sheet_name = match sheet { + Some(name) => name.to_string(), + None => workbook + .sheet_names() + .first() + .cloned() + .ok_or_else(|| read_err(path, "workbook has no sheets"))?, + }; + let range = workbook + .worksheet_range(&sheet_name) + .map_err(|e| read_err(path, e))?; + + let mut rows = range.rows(); + let headers: Vec = match rows.next() { + Some(first) => first.iter().map(cell_to_string).collect(), + None => return Ok(DataFrame::empty().lazy()), + }; + + let mut columns: Vec> = vec![Vec::new(); headers.len()]; + for row in rows { + for (idx, col) in columns.iter_mut().enumerate() { + let cell = row.get(idx).map(cell_to_string).unwrap_or_default(); + col.push(cell); + } + } + + let series: Vec = headers + .into_iter() + .zip(columns) + .map(|(name, values)| Series::new(name.into(), values).into_column()) + .collect(); + + DataFrame::new(series) + .map(|df| df.lazy()) + .map_err(|e| read_err(path, e)) +} + +fn cell_to_string(cell: &calamine::Data) -> String { + use calamine::Data; + match cell { + Data::Empty => String::new(), + Data::String(s) => s.clone(), + Data::Float(f) => f.to_string(), + Data::Int(i) => i.to_string(), + Data::Bool(b) => b.to_string(), + Data::DateTime(d) => d.to_string(), + other => other.to_string(), + } +} +``` + +> Note: `Series::new` takes a name as `PlSmallStr` in current Polars (`name.into()`), and `.into_column()` converts to the `Column` type used by `DataFrame::new`. If your pinned version still uses `Series` directly in `DataFrame::new`, drop `.into_column()` and collect `Vec`. The compiler will indicate which. `calamine::Data` variants may vary slightly by version — match what the error lists. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::scan_excel_reads_existing_fixture` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs Cargo.toml Cargo.lock +git commit -m "feat: PolarsEngine Excel scan via calamine" +``` + +--- + +### Task 9: Union-by-name concatenation (schema evolution) + +**Files:** +- Modify: `src/polars_engine.rs` + +> This replaces DuckDB's `UNION ALL BY NAME`: frames with differing/overlapping columns are aligned by name, missing columns filled with null. + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn concat_by_name_aligns_differing_columns() { + let engine = PolarsEngine::new(); + let a = df!["id" => [1i64], "name" => ["alice"]].unwrap().lazy(); + let b = df!["id" => [2i64], "extra" => ["x"]].unwrap().lazy(); + + let merged = engine.concat_by_name(vec![a, b]).unwrap(); + let df = engine.collect(merged).unwrap(); + + assert_eq!(df.height(), 2); + let mut names = df.get_column_names_str(); + names.sort(); + assert_eq!(names, vec!["extra", "id", "name"]); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --lib polars_engine::tests::concat_by_name_aligns_differing_columns` +Expected: FAIL — `concat_by_name` not defined. + +- [ ] **Step 3: Implement concat_by_name** + +Add to `impl PolarsEngine`: +```rust + /// Concatenate frames union-by-name (diagonal), filling missing columns with null. + pub fn concat_by_name(&self, frames: Vec) -> Result { + if frames.is_empty() { + return Err(read_err("(concat)", "no frames to concatenate")); + } + concat_lf_diagonal(frames, UnionArgs::default()).map_err(|e| read_err("(concat)", e)) + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::concat_by_name_aligns_differing_columns` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: PolarsEngine union-by-name concat (schema evolution)" +``` + +--- + +### Task 10: SQL execution via SQLContext (the user-facing SQL surface) + +**Files:** +- Modify: `src/polars_engine.rs` + +> This is the heart of the conversion: `_`, reference tables, `--where`/`--filter-sql`/`--post-sql` all run here. It must return clean `Result` errors (never hang) on unsupported SQL — the originating motivation. + +- [ ] **Step 1: Write the failing tests** + +```rust + #[test] + fn run_sql_filters_and_joins() { + let engine = PolarsEngine::new(); + let base = df!["id" => [1i64, 2], "region_id" => [10i64, 20]].unwrap().lazy(); + let regions = df!["id" => [10i64, 20], "region_name" => ["EMEA", "APAC"]].unwrap().lazy(); + + let out = engine + .run_sql( + base, + &[("regions".to_string(), regions)], + "SELECT _.id, r.region_name FROM _ JOIN regions r ON _.region_id = r.id WHERE _.id = 1", + ) + .unwrap(); + let df = engine.collect(out).unwrap(); + + assert_eq!(df.height(), 1); + assert_eq!(df.column("region_name").unwrap().str().unwrap().get(0), Some("EMEA")); + } + + #[test] + fn run_sql_returns_error_on_unsupported_sql_without_hanging() { + let engine = PolarsEngine::new(); + let base = df!["id" => [1i64]].unwrap().lazy(); + let result = engine.run_sql(base, &[], "DELETE FROM _ WHERE id = 1"); + assert!(matches!(result, Err(DtooError::Sql { .. }))); + } +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test --lib polars_engine::tests::run_sql` +Expected: FAIL — `run_sql` not defined. + +- [ ] **Step 3: Implement run_sql** + +Add to `impl PolarsEngine`: +```rust + /// Run user SQL with `base` registered as the magic table `_` and each ref + /// registered under its name. Returns the resulting LazyFrame. + pub fn run_sql( + &self, + base: LazyFrame, + refs: &[(String, LazyFrame)], + sql: &str, + ) -> Result { + let mut ctx = polars::sql::SQLContext::new(); + ctx.register("_", base); + for (name, lf) in refs { + ctx.register(name, lf.clone()); + } + ctx.execute(sql).map_err(|e| sql_err(sql, e)) + } +``` + +> Note: `polars::sql::SQLContext` requires the `sql` feature (added in Task 1). `execute` returns `PolarsResult`; parse/unsupported errors are returned here, and any execution errors surface on the later `collect()` — neither hangs. If `register` returns a `Result` in your pinned version, propagate it; if it returns `()`, the above is correct. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test --lib polars_engine::tests::run_sql` +Expected: PASS (both tests). + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: PolarsEngine SQL execution via SQLContext" +``` + +--- + +### Task 11: Schema introspection without materializing + +**Files:** +- Modify: `src/polars_engine.rs` + +> Used by dry-run, explicit `--schema` coercion, and profiling. + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn schema_of_returns_names_and_types() { + let engine = PolarsEngine::new(); + let lf = df!["id" => [1i64], "name" => ["alice"]].unwrap().lazy(); + + let schema = engine.schema_of(&lf).unwrap(); + let names: Vec<&str> = schema.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["id", "name"]); + assert_eq!(schema[0].1, DataType::Int64); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --lib polars_engine::tests::schema_of_returns_names_and_types` +Expected: FAIL — `schema_of` not defined. + +- [ ] **Step 3: Implement schema_of** + +Add to `impl PolarsEngine`: +```rust + /// Return (column name, dtype) pairs without materializing the frame. + pub fn schema_of(&self, lf: &LazyFrame) -> Result, DtooError> { + let schema = lf + .clone() + .collect_schema() + .map_err(|e| DtooError::Schema { message: e.to_string() })?; + Ok(schema + .iter() + .map(|(name, dtype)| (name.to_string(), dtype.clone())) + .collect()) + } +``` + +> Note: `collect_schema` returns a `SchemaRef`; iterating yields `(&PlSmallStr, &DataType)`. If your version exposes `.schema()` instead, use that. Adjust `.iter()` destructuring if the field tuple differs. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::schema_of_returns_names_and_types` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "feat: PolarsEngine lazy schema introspection" +``` + +--- + +### Task 12: Malformed input returns a clear error (the motivating bug) + +**Files:** +- Modify: `src/polars_engine.rs` + +> This is the regression test that encodes *why* this whole conversion exists: a bad field must produce a `Result` error, not a silent hang. + +- [ ] **Step 1: Write the failing/asserting test** + +```rust + #[test] + fn malformed_csv_surfaces_error_not_hang() { + // Ragged rows: header declares 2 columns, a data row has 4. + let path = tmp("bad", "csv"); + std::fs::write(&path, "id,name\n1,alice\n2,bob,extra,boom\n").unwrap(); + let engine = PolarsEngine::new(); + + let result = engine + .scan(path.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .and_then(|lf| engine.collect(lf)); + + assert!( + matches!(result, Err(DtooError::FileProcess { .. })), + "expected a FileProcess error for malformed CSV, got: {result:?}" + ); + let _ = std::fs::remove_file(path); + } +``` + +- [ ] **Step 2: Run test to verify behaviour** + +Run: `cargo test --lib polars_engine::tests::malformed_csv_surfaces_error_not_hang` +Expected: One of two outcomes: +- **PASS** — Polars already errors on ragged rows. Done; keep as a regression guard. +- **FAIL because the read silently succeeded** (Polars truncated/ignored extra fields) — proceed to Step 3 to enforce strictness. + +- [ ] **Step 3: Enforce strict parsing if needed** + +If Step 2 showed the malformed row was silently accepted, make CSV parsing strict by disabling error-tolerance explicitly. In the `InputFormat::Csv` arm of `scan`: +```rust + InputFormat::Csv { delimiter } => LazyCsvReader::new(path) + .with_separator(*delimiter as u8) + .with_has_header(true) + .with_ignore_errors(false) + .finish() + .map_err(|e| read_err(path, e)), +``` +If a different field count still does not raise (Polars is lenient about extra trailing fields by design), change the fixture to a type-violating value that Polars *does* reject under an inferred schema (e.g. header `id,amount` with a row `1,not_a_number` after forcing a numeric dtype), OR accept that this class is caught at the SQL/collect stage and adjust the assertion to assert the error surfaces at `collect`. The non-negotiable requirement: **a Result error, never a hang.** + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::malformed_csv_surfaces_error_not_hang` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "test: malformed input surfaces a clear error, never hangs" +``` + +--- + +### Task 13: Cloud-path guard (deferred-feature behaviour) + +**Files:** +- Modify: `src/polars_engine.rs` + +> Cloud is deferred (spec 34). Cloud paths must fail fast with a clear message rather than silently doing nothing — closing the previous silent-failure class. + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn cloud_paths_are_rejected_with_clear_error() { + let engine = PolarsEngine::new(); + let result = engine.scan("s3://bucket/data.parquet", &InputFormat::Parquet); + match result { + Err(DtooError::Config { message }) => { + assert!(message.contains("cloud storage")); + assert!(message.contains("not supported")); + } + other => panic!("expected Config error, got {other:?}"), + } + } +``` + +- [ ] **Step 2: Run test to verify it passes** + +Run: `cargo test --lib polars_engine::tests::cloud_paths_are_rejected_with_clear_error` +Expected: PASS already — `reject_cloud` was added in Task 2 and is called at the top of `scan`. If it fails, ensure `scan` calls `reject_cloud(path)?` before the `match`. + +- [ ] **Step 3: Commit** + +```bash +git add src/polars_engine.rs +git commit -m "test: cloud paths rejected with clear deferred-feature error" +``` + +--- + +### Task 14: Phase-1 gate — full suite green and clippy clean + +**Files:** none (verification only) + +- [ ] **Step 1: Run the whole test suite** + +Run: `cargo test` +Expected: ALL tests pass — the new `polars_engine` unit tests AND every pre-existing test (the DuckDB pipeline is untouched). + +- [ ] **Step 2: Clippy clean** + +Run: `cargo clippy --all-targets -- -D warnings` +Expected: no warnings. (`#![allow(dead_code)]` covers the not-yet-wired engine; it is removed in Phase 2.) + +- [ ] **Step 3: Format** + +Run: `cargo fmt --check` +Expected: clean (run `cargo fmt` and commit if not). + +- [ ] **Step 4: Commit any formatting** + +```bash +git add -A +git commit -m "style: cargo fmt after Phase 1" || echo "nothing to format" +``` + +--- + +## Self-Review Notes (coverage against spec 34) + +- **Engine API** (`scan`, `concat_by_name`, `run_sql`, `schema_of`, `collect`, `row_count`, `write`) — Tasks 2–11. ✅ +- **Readers** CSV/Parquet/NDJSON/Excel — Tasks 2, 3, 5, 8. ✅ +- **Glob** — provided natively by the Polars scanners used in `scan` (Parquet/CSV/NDJSON); no separate task needed. Excel does not glob (single file), matching today's behaviour. +- **Union-by-name** — Task 9. ✅ +- **User SQL via SQLContext + clean errors** — Task 10. ✅ +- **Writers + stdout sink** — Tasks 4, 5, 6. ✅ +- **CSV/NDJSON compression (flate2/zstd)** — Task 7. ✅ +- **Schema introspection** — Task 11. ✅ +- **Clear errors / no hang (motivating bug)** — Task 12. ✅ +- **Cloud deferral guard** — Task 13. ✅ +- **Profiling stat helpers, masking/lineage/limit as expressions, and the pipeline cutover + DuckDB removal** — intentionally **Phase 2** (separate plan), written against this phase's realized API. + +## Handoff to Phase 2 (separate plan, written after this compiles) + +Phase 2 — "Polars Engine Cutover" — will, slice by slice with the existing behavioural tests as the safety net: +1. Rewire `query_pipeline.rs` onto `PolarsEngine` (scan → concat → run_sql → write). +2. Reimplement `masking.rs`, `lineage.rs`, and `--limit` as native Polars expressions (no SQL). +3. Port `profiler.rs` aggregates to Polars expressions. +4. Port `crypto.rs` decrypt/encrypt application to expressions. +5. Rewire `reference_tables.rs`, `schema.rs`, `inspect.rs`, `profile_command.rs`, `convert_command.rs`. +6. Remove `duckdb` and `glob` from `Cargo.toml`, delete dead DuckDB code, remove `#![allow(dead_code)]`, and update `DESIGN.md` + the user guide (SQL limitations, cloud deferral). From f1463ba307b697693469112b9916337b2fa8dcc8 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 17:00:59 +0100 Subject: [PATCH 04/30] feat: scaffold PolarsEngine module alongside DuckDB engine Add polars (lazy, sql, csv, parquet, json, strings, dtype-full), calamine, flate2, and zstd dependencies; create the empty PolarsEngine skeleton in src/polars_engine.rs; register it in main.rs. Also fix a pre-existing clippy::collapsible_match lint in config.rs that became a -D warnings failure after the new clippy run. Co-Authored-By: Claude Sonnet 4.6 --- Cargo.lock | 2056 +++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 4 + src/config.rs | 9 +- src/main.rs | 2 + src/polars_engine.rs | 43 + 5 files changed, 2070 insertions(+), 44 deletions(-) create mode 100644 src/polars_engine.rs diff --git a/Cargo.lock b/Cargo.lock index afb0b34..1b156f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,7 +26,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -77,6 +77,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -142,6 +163,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "ar_archive_writer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -151,6 +181,28 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "argminmax" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70f13d10a41ac8d2ec79ee34178d61e6f47a29c2edfe7ef1721c7383b0359e65" +dependencies = [ + "half", + "num-traits", +] + +[[package]] +name = "array-init-cursor" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed51fe0f224d1d4ea768be38c51f9f831dee9d05c163c11fba0b8c44387b1fc3" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" @@ -320,6 +372,51 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "atoi" version = "2.0.0" @@ -329,6 +426,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi_simd" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ad17c7c205c2c28b527b9845eeb91cf1b4d008b438f98ce0e628227a822758e" +dependencies = [ + "debug_unsafe", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -347,11 +453,34 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +dependencies = [ + "serde_core", +] [[package]] name = "bitvec" @@ -365,6 +494,20 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -407,6 +550,33 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "boxcar" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f64beae40a84da1b4b26ff2761a5b895c12adc41dc25aaee1c4f2bbfe97a6e" + +[[package]] +name = "brotli" +version = "8.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -435,11 +605,57 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "calamine" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8822fe6253ca47aa5ad9a3be09f6fe7cd20c6a74e41b0aa42e8f4e3d523508df" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip 7.2.0", +] [[package]] name = "cast" @@ -447,6 +663,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.60" @@ -471,6 +696,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -482,7 +718,17 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", ] [[package]] @@ -535,6 +781,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -552,6 +807,30 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console" version = "0.15.11" @@ -585,6 +864,22 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -600,6 +895,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -609,6 +913,49 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crossterm" version = "0.29.0" @@ -658,6 +1005,12 @@ dependencies = [ "cipher", ] +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -706,14 +1059,17 @@ dependencies = [ "aes", "aes-gcm", "base64", + "calamine", "chrono", "clap", "comfy-table", "duckdb", "ecb", + "flate2", "glob", "hex", "indicatif", + "polars", "serde", "serde_json", "serde_yaml", @@ -721,6 +1077,7 @@ dependencies = [ "sha2", "thiserror", "uuid", + "zstd", ] [[package]] @@ -740,6 +1097,12 @@ dependencies = [ "strum", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "ecb" version = "0.1.2" @@ -749,12 +1112,27 @@ dependencies = [ "cipher", ] +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "encode_unicode" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -771,6 +1149,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ethnum" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -784,10 +1189,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] -name = "filetime" -version = "0.2.27" +name = "fast-float2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" dependencies = [ "cfg-if", "libc", @@ -811,12 +1228,33 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -826,12 +1264,37 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -848,12 +1311,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -872,8 +1357,10 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -927,6 +1414,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -947,18 +1435,49 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if", "crunchy", "num-traits", + "serde", "zerocopy", ] +[[package]] +name = "halfbrown" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ed2f2edad8a14c8186b847909a41fbb9c3eafa44f88bd891114ed5019da09" +dependencies = [ + "hashbrown 0.16.1", + "serde", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -974,7 +1493,9 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "allocator-api2", + "equivalent", + "foldhash 0.1.5", ] [[package]] @@ -982,6 +1503,14 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "rayon", + "serde", + "serde_core", +] [[package]] name = "hashbrown" @@ -1010,6 +1539,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.4.0" @@ -1049,6 +1587,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "hyper" version = "1.9.0" @@ -1059,6 +1603,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1079,6 +1624,7 @@ dependencies = [ "hyper", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1121,7 +1667,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -1299,6 +1845,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1327,6 +1882,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonpath_lib_polars_vendor" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4bd9354947622f7471ff713eacaabdb683ccb13bba4edccaab9860abf480b7d" +dependencies = [ + "log", + "serde", + "serde_json", +] + [[package]] name = "leb128fmt" version = "0.1.0" @@ -1410,7 +1976,7 @@ dependencies = [ "serde_json", "tar", "vcpkg", - "zip", + "zip 6.0.0", ] [[package]] @@ -1470,12 +2036,40 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1497,6 +2091,24 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "now" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89e9874397a1f0a52fc1f197a8effd9735223cb2390e9dcc83ac6cd02923d0" +dependencies = [ + "chrono", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1516,6 +2128,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1541,6 +2164,71 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.1", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1559,6 +2247,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1579,7 +2279,7 @@ dependencies = [ "libc", "redox_syscall 0.5.18", "smallvec", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -1588,6 +2288,24 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1607,35 +2325,691 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] -name = "polyval" -version = "0.6.2" +name = "planus" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +checksum = "3daf8e3d4b712abe1d690838f6e29fb76b76ea19589c4afa39ec30e12f62af71" dependencies = [ - "cfg-if", - "cpufeatures", - "opaque-debug", - "universal-hash", + "array-init-cursor", + "hashbrown 0.15.5", ] [[package]] -name = "portable-atomic" -version = "1.13.1" +name = "polars" +version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "82f1f122456ec136102033b13f71905b7c3f01e526642679c86aace9f9cdefde" +dependencies = [ + "getrandom 0.2.17", + "getrandom 0.3.4", + "polars-arrow", + "polars-buffer", + "polars-compute", + "polars-core", + "polars-error", + "polars-io", + "polars-lazy", + "polars-ops", + "polars-parquet", + "polars-plan", + "polars-sql", + "polars-time", + "polars-utils", + "version_check", +] [[package]] -name = "potential_utf" -version = "0.1.5" +name = "polars-arrow" +version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "87d4892d5cc6461bb4a184d18e6fa03a5d316ee1d6de06a33dfa08d479fbc2db" dependencies = [ - "zerovec", + "atoi_simd", + "bitflags", + "bytemuck", + "bytes", + "chrono", + "chrono-tz", + "dyn-clone", + "either", + "ethnum", + "getrandom 0.2.17", + "getrandom 0.3.4", + "half", + "hashbrown 0.16.1", + "itoa", + "lz4", + "num-traits", + "polars-arrow-format", + "polars-buffer", + "polars-error", + "polars-schema", + "polars-utils", + "serde", + "simdutf8", + "streaming-iterator", + "strum_macros", + "version_check", + "zstd", ] [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "polars-arrow-format" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a556ac0ee744e61e167f34c1eb0013ce740e0ee6cd8c158b2ec0b518f10e6675" +dependencies = [ + "planus", + "serde", +] + +[[package]] +name = "polars-async" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e87f836190486f500b28347436985cc0af29b7a514e53f98840d396ce4d5f5" +dependencies = [ + "atomic-waker", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "parking_lot", + "pin-project-lite", + "polars-config", + "polars-error", + "polars-utils", + "rand 0.9.3", + "slotmap", + "tokio", +] + +[[package]] +name = "polars-buffer" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e481eeaf33c544ac0dd71a2e375553ca2fdae47b3472a96eaccb6eb43218783d" +dependencies = [ + "bytemuck", + "either", + "polars-utils", + "serde", + "version_check", +] + +[[package]] +name = "polars-compute" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55d41642a9ee887ac394c5a310af3256fa8340a86cde2cb624c515aa963461c" +dependencies = [ + "atoi_simd", + "bytemuck", + "chrono", + "either", + "fast-float2", + "half", + "hashbrown 0.16.1", + "itoa", + "num-traits", + "polars-arrow", + "polars-buffer", + "polars-error", + "polars-utils", + "rand 0.9.3", + "serde", + "strength_reduce", + "strum_macros", + "version_check", + "zmij", +] + +[[package]] +name = "polars-config" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65af861341b00eac73bcb65423fb5cc3d2322526d6b7561a0ddf094947c38033" +dependencies = [ + "polars-error", + "serde", +] + +[[package]] +name = "polars-core" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e5924fc46306054bae78f9d35ea5e404cf185baa7f170eb55a16ff95191069c" +dependencies = [ + "bitflags", + "boxcar", + "bytemuck", + "chrono", + "chrono-tz", + "comfy-table", + "either", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "indexmap", + "itoa", + "num-traits", + "polars-arrow", + "polars-async", + "polars-buffer", + "polars-compute", + "polars-config", + "polars-dtype", + "polars-error", + "polars-row", + "polars-schema", + "polars-utils", + "rand 0.9.3", + "rand_distr", + "rayon", + "regex", + "serde", + "serde_json", + "strum_macros", + "tokio", + "uuid", + "version_check", + "xxhash-rust", +] + +[[package]] +name = "polars-dtype" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b65a750bb99ea66be90c8a7e336f6f3a87427a0f7f89d2a40adae98314e9b27" +dependencies = [ + "boxcar", + "hashbrown 0.16.1", + "polars-arrow", + "polars-error", + "polars-utils", + "serde", + "uuid", +] + +[[package]] +name = "polars-error" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49a75e3406b9b5b4e5ff177877fe0de766e9688fbdb263a7b25f293dc47d61a" +dependencies = [ + "object_store", + "parking_lot", + "polars-arrow-format", + "regex", + "signal-hook", + "simdutf8", +] + +[[package]] +name = "polars-expr" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21fdd37e8d9ef109f13d3454baffa0a57041cf60069123b8a2bd846c8ad0205" +dependencies = [ + "bitflags", + "hashbrown 0.16.1", + "num-traits", + "polars-arrow", + "polars-buffer", + "polars-compute", + "polars-core", + "polars-io", + "polars-json", + "polars-ops", + "polars-plan", + "polars-row", + "polars-time", + "polars-utils", + "rand 0.9.3", + "rayon", + "recursive", + "regex", + "version_check", +] + +[[package]] +name = "polars-io" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6363a1c44a65fe8d73cce7fe4d77c9b6fea3a0da44007012e755e5b4e65aa078" +dependencies = [ + "async-trait", + "atoi_simd", + "blake3", + "bytes", + "chrono", + "fast-float2", + "fastrand", + "fs4", + "futures", + "glob", + "hashbrown 0.16.1", + "home", + "itoa", + "memchr", + "memmap2", + "num-traits", + "object_store", + "parking_lot", + "percent-encoding", + "polars-arrow", + "polars-buffer", + "polars-compute", + "polars-config", + "polars-core", + "polars-error", + "polars-json", + "polars-parquet", + "polars-schema", + "polars-time", + "polars-utils", + "rand 0.9.3", + "rayon", + "regex", + "reqwest", + "serde", + "serde_json", + "simd-json", + "simdutf8", + "tokio", + "zmij", +] + +[[package]] +name = "polars-json" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dca6cd170370ef7e189a4c362846c57553843653c3fe65aafe12ca77599987c" +dependencies = [ + "chrono", + "fallible-streaming-iterator", + "hashbrown 0.16.1", + "indexmap", + "itoa", + "num-traits", + "polars-arrow", + "polars-compute", + "polars-error", + "polars-utils", + "simd-json", + "streaming-iterator", + "zmij", +] + +[[package]] +name = "polars-lazy" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809d9590232a37d638337629c18279af97bdb0d17c3d8b2b6bb186e903e8bd5e" +dependencies = [ + "bitflags", + "chrono", + "either", + "memchr", + "polars-arrow", + "polars-buffer", + "polars-compute", + "polars-config", + "polars-core", + "polars-expr", + "polars-io", + "polars-json", + "polars-mem-engine", + "polars-ops", + "polars-plan", + "polars-stream", + "polars-time", + "polars-utils", + "rayon", + "version_check", +] + +[[package]] +name = "polars-mem-engine" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55c6b7d162c506bc8eee82b065fa0399ebcd20b8f08675a534f3d360904ba38" +dependencies = [ + "memmap2", + "polars-arrow", + "polars-core", + "polars-error", + "polars-expr", + "polars-io", + "polars-json", + "polars-ops", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", + "recursive", +] + +[[package]] +name = "polars-ooc" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3eea0b386837b760a97ec9c92df99cbc10f94885cae060fd7100f9b794163" +dependencies = [ + "async-trait", + "boxcar", + "libc", + "polars-async", + "polars-config", + "polars-core", + "polars-io", + "polars-utils", + "thread_local", + "tokio", +] + +[[package]] +name = "polars-ops" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb146490a717ac5ae4ff3a22a5adf3ebae79361f187b1f550f9e24783d7ad765" +dependencies = [ + "argminmax", + "base64", + "bytemuck", + "chrono", + "chrono-tz", + "either", + "hashbrown 0.16.1", + "hex", + "indexmap", + "jsonpath_lib_polars_vendor", + "libm", + "memchr", + "num-traits", + "polars-arrow", + "polars-buffer", + "polars-compute", + "polars-core", + "polars-error", + "polars-json", + "polars-schema", + "polars-utils", + "rayon", + "regex", + "regex-syntax", + "serde_json", + "strum_macros", + "unicode-normalization", + "unicode-reverse", + "version_check", +] + +[[package]] +name = "polars-parquet" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6b79ba2103c00cbb9c5dd4459ffff1d8ce15286c7a6d376a04c711df20d8b7" +dependencies = [ + "async-stream", + "base64", + "brotli", + "bytemuck", + "ethnum", + "flate2", + "futures", + "hashbrown 0.16.1", + "lz4", + "num-traits", + "polars-arrow", + "polars-buffer", + "polars-compute", + "polars-config", + "polars-error", + "polars-parquet-format", + "polars-utils", + "regex", + "serde", + "simdutf8", + "snap", + "streaming-decompression", + "zstd", +] + +[[package]] +name = "polars-parquet-format" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c025243dcfe8dbc57e94d9f82eb3bef10b565ab180d5b99bed87fd8aea319ce1" +dependencies = [ + "async-trait", + "futures", +] + +[[package]] +name = "polars-plan" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5ccc230515adb10762a8c7b0df03fd88f3328deb5b60e9b1eeb2eceef4d344" +dependencies = [ + "bitflags", + "blake3", + "bytemuck", + "bytes", + "chrono", + "chrono-tz", + "either", + "futures", + "hashbrown 0.16.1", + "indexmap", + "memmap2", + "num-traits", + "percent-encoding", + "polars-arrow", + "polars-buffer", + "polars-compute", + "polars-config", + "polars-core", + "polars-error", + "polars-io", + "polars-json", + "polars-ops", + "polars-parquet", + "polars-time", + "polars-utils", + "rayon", + "recursive", + "regex", + "sha2", + "slotmap", + "strum_macros", + "tokio", + "version_check", +] + +[[package]] +name = "polars-row" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d4e3254450024078e10c919ecd3b467bdcfdd5cf386c2ca6eedec89bd4771d2" +dependencies = [ + "bitflags", + "bytemuck", + "polars-arrow", + "polars-buffer", + "polars-compute", + "polars-dtype", + "polars-error", + "polars-utils", +] + +[[package]] +name = "polars-schema" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8a0de8951d02576fd0cdcecd9c605a6b6364d3105b7469b8d7874ea34eea2f" +dependencies = [ + "indexmap", + "polars-error", + "polars-utils", + "serde", + "version_check", +] + +[[package]] +name = "polars-sql" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b282a6164927eb12774b66b071b773a1573173ae53758e8d4df50389ff06efa2" +dependencies = [ + "bitflags", + "hex", + "polars-core", + "polars-error", + "polars-lazy", + "polars-ops", + "polars-plan", + "polars-time", + "polars-utils", + "regex", + "serde", + "sqlparser", +] + +[[package]] +name = "polars-stream" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfa8ff4ee21799898579595a0ef2fb728d0a9cac3d061835fb7f7f6dd854734a" +dependencies = [ + "async-channel", + "async-trait", + "bitflags", + "bytes", + "chrono-tz", + "crossbeam-channel", + "crossbeam-queue", + "futures", + "memchr", + "num-traits", + "parking_lot", + "percent-encoding", + "polars-arrow", + "polars-async", + "polars-buffer", + "polars-compute", + "polars-config", + "polars-core", + "polars-error", + "polars-expr", + "polars-io", + "polars-json", + "polars-mem-engine", + "polars-ooc", + "polars-ops", + "polars-parquet", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", + "recursive", + "slotmap", + "tokio", + "uuid", + "version_check", +] + +[[package]] +name = "polars-time" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1063fe074c4212a54917be604377c6e6bfbc8b6c942a5c57be214e4ccaaafdf" +dependencies = [ + "atoi_simd", + "bytemuck", + "chrono", + "chrono-tz", + "now", + "num-traits", + "polars-arrow", + "polars-compute", + "polars-core", + "polars-error", + "polars-ops", + "polars-utils", + "rayon", + "regex", + "strum_macros", +] + +[[package]] +name = "polars-utils" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590b0a94aa8f97992d52f1198600ecc1c1f7cfa03c1b31cae057143455804ac0" +dependencies = [ + "argminmax", + "bincode", + "bytemuck", + "bytes", + "compact_str", + "either", + "flate2", + "foldhash 0.2.0", + "futures", + "half", + "hashbrown 0.16.1", + "indexmap", + "libc", + "memmap2", + "num-derive", + "num-traits", + "polars-config", + "polars-error", + "rand 0.9.3", + "raw-cpuid", + "rayon", + "regex", + "rmp-serde", + "serde", + "serde_json", + "serde_stacker", + "slotmap", + "stacker", + "sysinfo", + "tokio", + "uuid", + "version_check", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ @@ -1670,6 +3044,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "ptr_meta" version = "0.1.4" @@ -1690,6 +3074,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "encoding_rs", + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1793,6 +3188,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -1831,6 +3237,71 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.3", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1849,6 +3320,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "regex" version = "1.12.3" @@ -1898,6 +3389,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", @@ -1910,6 +3402,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -1917,12 +3410,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots", ] @@ -1970,6 +3465,25 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "rust_decimal" version = "1.41.0" @@ -2020,6 +3534,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -2053,6 +3579,24 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2065,6 +3609,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -2107,6 +3674,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -2114,6 +3682,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_stacker" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a" +dependencies = [ + "serde", + "serde_core", + "stacker", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2146,7 +3725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2154,18 +3733,38 @@ dependencies = [ name = "sha2" version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" dependencies = [ - "cfg-if", - "cpufeatures", - "digest", + "libc", + "signal-hook-registry", ] [[package]] -name = "shlex" -version = "1.3.0" +name = "signal-hook-registry" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] [[package]] name = "simd-adler32" @@ -2173,24 +3772,61 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd-json" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4255126f310d2ba20048db6321c81ab376f6a6735608bf11f0785c41f01f64e3" +dependencies = [ + "ahash 0.8.12", + "halfbrown", + "once_cell", + "ref-cast", + "serde", + "serde_json", + "simdutf8", + "value-trait", +] + [[package]] name = "simdutf8" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "socket2" version = "0.6.3" @@ -2201,12 +3837,74 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sqlparser" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "505aa16b045c4c1375bf5f125cce3813d0176325bfe9ffc4a903f423de7774ff" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028e551d5e270b31b9f3ea271778d9d827148d4287a5d96167b6bb9787f5cc38" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "streaming-decompression" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" +dependencies = [ + "fallible-streaming-iterator", +] + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strsim" version = "0.11.1" @@ -2282,6 +3980,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "sysinfo" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + [[package]] name = "tap" version = "1.0.1" @@ -2319,6 +4031,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2364,9 +4085,21 @@ dependencies = [ "mio", "pin-project-lite", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -2377,6 +4110,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -2459,9 +4205,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -2477,6 +4235,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.19.0" @@ -2489,6 +4253,24 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-reverse" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6f4888ebc23094adfb574fdca9fdc891826287a6397d2cd28802ffd6f20c76" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "unicode-segmentation" version = "1.13.2" @@ -2529,6 +4311,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -2561,9 +4349,22 @@ checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "getrandom 0.4.2", "js-sys", + "serde_core", "wasm-bindgen", ] +[[package]] +name = "value-trait" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e80f0c733af0720a501b3905d22e2f97662d8eacfe082a75ed7ffb5ab08cb59" +dependencies = [ + "float-cmp", + "halfbrown", + "itoa", + "ryu", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2576,6 +4377,22 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2687,6 +4504,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -2744,12 +4574,56 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core 0.61.2", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -2758,9 +4632,20 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", ] [[package]] @@ -2785,19 +4670,53 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -2806,7 +4725,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2842,7 +4761,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2867,7 +4786,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link", + "windows-link 0.2.1", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -2878,6 +4797,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -3096,6 +5024,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yoke" version = "0.8.2" @@ -3213,6 +5147,20 @@ dependencies = [ "zopfli", ] +[[package]] +name = "zip" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.3" @@ -3236,3 +5184,31 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 072f21f..3aeb921 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,10 @@ aes = "0.8" ecb = "0.1" sha1 = "0.10" base64 = "0.22" +polars = { version = "0.54.4", features = ["lazy", "sql", "csv", "parquet", "json", "strings", "dtype-full"] } +calamine = "0.35.0" +flate2 = "1.1.9" +zstd = "0.13.3" [profile.release] opt-level = 3 diff --git a/src/config.rs b/src/config.rs index dcf30a1..0c7d2d5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -150,11 +150,12 @@ fn validate_top_level_types(value: &serde_yaml::Value) -> Result<(), DtooError> "mask" => ensure_mapping_or_null(key, val)?, "profile" => ensure_mapping_or_null(key, val)?, "cloud" => ensure_mapping_or_null(key, val)?, - "crypto_profiles" => { - if !(val.is_null() || val.as_sequence().is_some() || val.as_mapping().is_some()) { - return Err(config_type_error(key, "list or mapping")); - } + "crypto_profiles" + if !(val.is_null() || val.as_sequence().is_some() || val.as_mapping().is_some()) => + { + return Err(config_type_error(key, "list or mapping")); } + "crypto_profiles" => {} _ => {} } } diff --git a/src/main.rs b/src/main.rs index 41ff766..3ac949a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,8 @@ mod output_writer; #[allow(dead_code)] mod path_utils; #[allow(dead_code)] +mod polars_engine; +#[allow(dead_code)] mod profile_command; #[allow(dead_code)] mod profiler; diff --git a/src/polars_engine.rs b/src/polars_engine.rs new file mode 100644 index 0000000..950a8c1 --- /dev/null +++ b/src/polars_engine.rs @@ -0,0 +1,43 @@ +//! Pure-Rust data engine built on Polars. Phase 1: built alongside the DuckDB +//! engine and not yet wired into the pipeline. See docs/specs/34-polars-engine.md. +// dead_code allowed via `#[allow(dead_code)]` on the `mod polars_engine` declaration in main.rs; +// removed in Phase 2 when the pipeline is rewired onto this engine. +#![allow(unused_imports)] + +use std::path::Path; + +use polars::prelude::*; + +use crate::engine::{CompressionCodec, ExportFormat, InputFormat}; +use crate::error::DtooError; + +/// Stateless handle for Polars-backed data operations. +pub struct PolarsEngine; + +impl PolarsEngine { + /// Construct a new engine handle. + pub fn new() -> Self { + Self + } +} + +fn read_err(path: &str, source: impl std::fmt::Display) -> DtooError { + DtooError::FileProcess { + path: path.to_string(), + message: source.to_string(), + } +} + +fn write_err(source: impl std::fmt::Display) -> DtooError { + DtooError::Output { + message: source.to_string(), + } +} + +fn sql_err(sql: &str, source: PolarsError) -> DtooError { + DtooError::Sql { + context: "polars-sql".to_string(), + sql: sql.to_string(), + source: Box::new(std::io::Error::other(source.to_string())), + } +} From ed32f36865cf5b2224743dc372cae19255ad4d85 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 17:05:35 +0100 Subject: [PATCH 05/30] feat: PolarsEngine CSV scan, collect, row_count Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 950a8c1..50c929f 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -19,6 +19,30 @@ impl PolarsEngine { pub fn new() -> Self { Self } + + /// Lazily scan an input file by format. Glob patterns are supported by the + /// underlying Polars scanners for Parquet/CSV/NDJSON. + pub fn scan(&self, path: &str, format: &InputFormat) -> Result { + reject_cloud(path)?; + match format { + InputFormat::Csv { delimiter } => LazyCsvReader::new(path.into()) + .with_separator(*delimiter as u8) + .with_has_header(true) + .finish() + .map_err(|e| read_err(path, e)), + _ => Err(read_err(path, "unsupported format (implemented in a later task)")), + } + } + + /// Materialize a LazyFrame into a DataFrame. + pub fn collect(&self, lf: LazyFrame) -> Result { + lf.collect().map_err(|e| read_err("(query)", e)) + } + + /// Count rows without retaining the materialized frame. + pub fn row_count(&self, lf: LazyFrame) -> Result { + Ok(self.collect(lf)?.height()) + } } fn read_err(path: &str, source: impl std::fmt::Display) -> DtooError { @@ -34,6 +58,15 @@ fn write_err(source: impl std::fmt::Display) -> DtooError { } } +fn reject_cloud(path: &str) -> Result<(), DtooError> { + if crate::path_utils::is_cloud_path(path) { + return Err(DtooError::Config { + message: format!("cloud storage ({path}) is not supported in this build yet"), + }); + } + Ok(()) +} + fn sql_err(sql: &str, source: PolarsError) -> DtooError { DtooError::Sql { context: "polars-sql".to_string(), @@ -41,3 +74,33 @@ fn sql_err(sql: &str, source: PolarsError) -> DtooError { source: Box::new(std::io::Error::other(source.to_string())), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp(name: &str, ext: &str) -> std::path::PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("dtoo-pe-{name}-{nanos}.{ext}")) + } + + #[test] + fn scan_csv_reads_rows_and_columns() { + let path = tmp("csv", "csv"); + std::fs::write(&path, "id,name\n1,alice\n2,bob\n").unwrap(); + let engine = PolarsEngine::new(); + + let lf = engine + .scan(path.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(); + let count = engine.row_count(lf.clone()).unwrap(); + let df = engine.collect(lf).unwrap(); + + assert_eq!(count, 2); + assert_eq!(df.get_column_names(), vec!["id", "name"]); + let _ = std::fs::remove_file(path); + } +} From 09e235fc3efebb961e2dce3a61e8df1c2d7d7eb7 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 18:13:42 +0100 Subject: [PATCH 06/30] feat: PolarsEngine NDJSON scan --- src/polars_engine.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 50c929f..15cb7cc 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -30,6 +30,9 @@ impl PolarsEngine { .with_has_header(true) .finish() .map_err(|e| read_err(path, e)), + InputFormat::Ndjson => LazyJsonLineReader::new(path.into()) + .finish() + .map_err(|e| read_err(path, e)), _ => Err(read_err(path, "unsupported format (implemented in a later task)")), } } @@ -87,6 +90,17 @@ mod tests { std::env::temp_dir().join(format!("dtoo-pe-{name}-{nanos}.{ext}")) } + #[test] + fn scan_ndjson_reads_rows() { + let path = tmp("ndjson", "ndjson"); + std::fs::write(&path, "{\"id\":1,\"name\":\"alice\"}\n{\"id\":2,\"name\":\"bob\"}\n").unwrap(); + let engine = PolarsEngine::new(); + + let lf = engine.scan(path.to_str().unwrap(), &InputFormat::Ndjson).unwrap(); + assert_eq!(engine.row_count(lf).unwrap(), 2); + let _ = std::fs::remove_file(path); + } + #[test] fn scan_csv_reads_rows_and_columns() { let path = tmp("csv", "csv"); From 4a46c890a9f5dff990d1baf0ecae57f15d3d3ad0 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 18:20:00 +0100 Subject: [PATCH 07/30] feat: PolarsEngine CSV writer with file/stdout sink Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 75 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 15cb7cc..064674a 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -46,6 +46,30 @@ impl PolarsEngine { pub fn row_count(&self, lf: LazyFrame) -> Result { Ok(self.collect(lf)?.height()) } + + /// Write a DataFrame to a file path or, when `dest` is None, to stdout. + pub fn write( + &self, + mut df: DataFrame, + dest: Option<&Path>, + format: ExportFormat, + header: bool, + delimiter: char, + compression: Option, + ) -> Result<(), DtooError> { + let sink = open_sink(dest)?; + match format { + ExportFormat::Csv => { + let writer = wrap_compression(sink, compression); + CsvWriter::new(writer) + .include_header(header) + .with_separator(delimiter as u8) + .finish(&mut df) + .map_err(write_err) + } + _ => Err(write_err("unsupported export format (implemented in a later task)")), + } + } } fn read_err(path: &str, source: impl std::fmt::Display) -> DtooError { @@ -70,6 +94,36 @@ fn reject_cloud(path: &str) -> Result<(), DtooError> { Ok(()) } +fn open_sink(dest: Option<&Path>) -> Result, DtooError> { + match dest { + Some(path) => { + let file = std::fs::File::create(path) + .map_err(|e| write_err(format!("{}: {e}", path.display())))?; + Ok(Box::new(file)) + } + None => Ok(Box::new(std::io::stdout())), + } +} + +fn wrap_compression( + sink: Box, + compression: Option, +) -> Box { + match compression { + None => sink, + Some(CompressionCodec::Gzip) => { + Box::new(flate2::write::GzEncoder::new(sink, flate2::Compression::default())) + } + Some(CompressionCodec::Zstd) => { + Box::new( + zstd::stream::write::Encoder::new(sink, 0) + .expect("zstd encoder init") + .auto_finish(), + ) + } + } +} + fn sql_err(sql: &str, source: PolarsError) -> DtooError { DtooError::Sql { context: "polars-sql".to_string(), @@ -117,4 +171,25 @@ mod tests { assert_eq!(df.get_column_names(), vec!["id", "name"]); let _ = std::fs::remove_file(path); } + + #[test] + fn write_csv_roundtrips() { + let src = tmp("wcsv-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("wcsv-dst", "csv"); + let engine = PolarsEngine::new(); + + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine + .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', None) + .unwrap(); + + let written = std::fs::read_to_string(&dst).unwrap(); + assert!(written.contains("id,name")); + assert!(written.contains("1,alice")); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } } From bb67cc5f028a11237f08ee690b80fb6a3c632d5c Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 19:02:32 +0100 Subject: [PATCH 08/30] feat: PolarsEngine Parquet scan and writer Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 064674a..3c750b5 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -33,6 +33,8 @@ impl PolarsEngine { InputFormat::Ndjson => LazyJsonLineReader::new(path.into()) .finish() .map_err(|e| read_err(path, e)), + InputFormat::Parquet => LazyFrame::scan_parquet(path.into(), ScanArgsParquet::default()) + .map_err(|e| read_err(path, e)), _ => Err(read_err(path, "unsupported format (implemented in a later task)")), } } @@ -67,6 +69,18 @@ impl PolarsEngine { .finish(&mut df) .map_err(write_err) } + ExportFormat::Parquet => { + let codec = match compression { + Some(CompressionCodec::Gzip) => ParquetCompression::Gzip(None), + Some(CompressionCodec::Zstd) => ParquetCompression::Zstd(None), + None => ParquetCompression::default(), + }; + ParquetWriter::new(sink) + .with_compression(codec) + .finish(&mut df) + .map(|_| ()) + .map_err(write_err) + } _ => Err(write_err("unsupported export format (implemented in a later task)")), } } @@ -172,6 +186,24 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn parquet_roundtrips() { + let src = tmp("pq-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n2,bob\n").unwrap(); + let pq = tmp("pq-mid", "parquet"); + let engine = PolarsEngine::new(); + + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine.write(df, Some(pq.as_path()), ExportFormat::Parquet, true, ',', None).unwrap(); + + let back = engine.scan(pq.to_str().unwrap(), &InputFormat::Parquet).unwrap(); + assert_eq!(engine.row_count(back).unwrap(), 2); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(pq); + } + #[test] fn write_csv_roundtrips() { let src = tmp("wcsv-src", "csv"); From f2cff763ca172eb86ffd61162c9ae58331fbfbf4 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 19:43:07 +0100 Subject: [PATCH 09/30] feat: PolarsEngine NDJSON writer Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 3c750b5..e188d9b 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -81,7 +81,13 @@ impl PolarsEngine { .map(|_| ()) .map_err(write_err) } - _ => Err(write_err("unsupported export format (implemented in a later task)")), + ExportFormat::Ndjson => { + let writer = wrap_compression(sink, compression); + JsonWriter::new(writer) + .with_json_format(JsonFormat::JsonLines) + .finish(&mut df) + .map_err(write_err) + } } } } @@ -204,6 +210,24 @@ mod tests { let _ = std::fs::remove_file(pq); } + #[test] + fn ndjson_write_roundtrips() { + let src = tmp("wnd-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("wnd-dst", "ndjson"); + let engine = PolarsEngine::new(); + + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine.write(df, Some(dst.as_path()), ExportFormat::Ndjson, true, ',', None).unwrap(); + + let back = engine.scan(dst.to_str().unwrap(), &InputFormat::Ndjson).unwrap(); + assert_eq!(engine.row_count(back).unwrap(), 1); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } + #[test] fn write_csv_roundtrips() { let src = tmp("wcsv-src", "csv"); From cf6390a9b31dc7ae177c71e77622a6a77ff5e7f6 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 19:54:19 +0100 Subject: [PATCH 10/30] feat: gzip/zstd compression for CSV/NDJSON output Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index e188d9b..7882787 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -248,4 +248,47 @@ mod tests { let _ = std::fs::remove_file(src); let _ = std::fs::remove_file(dst); } + + #[test] + fn csv_gzip_output_is_decompressible() { + let src = tmp("gz-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("gz-dst", "csv.gz"); + let engine = PolarsEngine::new(); + + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine + .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', Some(CompressionCodec::Gzip)) + .unwrap(); + + let bytes = std::fs::read(&dst).unwrap(); + let mut decoder = flate2::read::GzDecoder::new(&bytes[..]); + let mut text = String::new(); + std::io::Read::read_to_string(&mut decoder, &mut text).unwrap(); + assert!(text.contains("1,alice")); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } + + #[test] + fn csv_zstd_output_is_decompressible() { + let src = tmp("zs-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("zs-dst", "csv.zst"); + let engine = PolarsEngine::new(); + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine + .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', Some(CompressionCodec::Zstd)) + .unwrap(); + + let bytes = std::fs::read(&dst).unwrap(); + let text = String::from_utf8(zstd::stream::decode_all(&bytes[..]).unwrap()).unwrap(); + assert!(text.contains("1,alice")); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } } From 0b7029aefd820059a72e2a226baf0fb9c363e819 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 20:45:58 +0100 Subject: [PATCH 11/30] fix: finalize compression frames explicitly to surface write errors Replace wrap_compression (which silently swallowed finalize errors via Drop and panicked on zstd init) with write_with_optional_compression, which explicitly calls GzEncoder::finish() / zstd Encoder::finish() and maps errors to DtooError. Add Default impl for PolarsEngine and a test for the header=false CSV path. Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 73 +++++++++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 7882787..c831dd3 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -14,6 +14,12 @@ use crate::error::DtooError; /// Stateless handle for Polars-backed data operations. pub struct PolarsEngine; +impl Default for PolarsEngine { + fn default() -> Self { + Self::new() + } +} + impl PolarsEngine { /// Construct a new engine handle. pub fn new() -> Self { @@ -61,14 +67,14 @@ impl PolarsEngine { ) -> Result<(), DtooError> { let sink = open_sink(dest)?; match format { - ExportFormat::Csv => { - let writer = wrap_compression(sink, compression); - CsvWriter::new(writer) + ExportFormat::Csv => write_with_optional_compression(sink, compression, |w| { + CsvWriter::new(w) .include_header(header) .with_separator(delimiter as u8) .finish(&mut df) + .map(|_| ()) .map_err(write_err) - } + }), ExportFormat::Parquet => { let codec = match compression { Some(CompressionCodec::Gzip) => ParquetCompression::Gzip(None), @@ -81,13 +87,13 @@ impl PolarsEngine { .map(|_| ()) .map_err(write_err) } - ExportFormat::Ndjson => { - let writer = wrap_compression(sink, compression); - JsonWriter::new(writer) + ExportFormat::Ndjson => write_with_optional_compression(sink, compression, |w| { + JsonWriter::new(w) .with_json_format(JsonFormat::JsonLines) .finish(&mut df) + .map(|_| ()) .map_err(write_err) - } + }), } } } @@ -125,21 +131,35 @@ fn open_sink(dest: Option<&Path>) -> Result, DtooError> } } -fn wrap_compression( +/// Run `write_body` against a (possibly compressed) sink, then explicitly +/// finalize the compression frame so a failed final flush is surfaced as an +/// error rather than silently truncating the output. +fn write_with_optional_compression( sink: Box, compression: Option, -) -> Box { + write_body: F, +) -> Result<(), DtooError> +where + F: FnOnce(&mut dyn std::io::Write) -> Result<(), DtooError>, +{ match compression { - None => sink, + None => { + let mut sink = sink; + write_body(&mut sink)?; + sink.flush().map_err(write_err) + } Some(CompressionCodec::Gzip) => { - Box::new(flate2::write::GzEncoder::new(sink, flate2::Compression::default())) + let mut encoder = + flate2::write::GzEncoder::new(sink, flate2::Compression::default()); + write_body(&mut encoder)?; + encoder.finish().map_err(write_err)?; + Ok(()) } Some(CompressionCodec::Zstd) => { - Box::new( - zstd::stream::write::Encoder::new(sink, 0) - .expect("zstd encoder init") - .auto_finish(), - ) + let mut encoder = zstd::stream::write::Encoder::new(sink, 0).map_err(write_err)?; + write_body(&mut encoder)?; + encoder.finish().map_err(write_err)?; + Ok(()) } } } @@ -249,6 +269,25 @@ mod tests { let _ = std::fs::remove_file(dst); } + #[test] + fn write_csv_without_header_omits_header_row() { + let src = tmp("noh-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("noh-dst", "csv"); + let engine = PolarsEngine::new(); + let df = engine + .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .unwrap(); + engine + .write(df, Some(dst.as_path()), ExportFormat::Csv, false, ',', None) + .unwrap(); + let written = std::fs::read_to_string(&dst).unwrap(); + assert!(!written.contains("id,name")); + assert!(written.contains("1,alice")); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } + #[test] fn csv_gzip_output_is_decompressible() { let src = tmp("gz-src", "csv"); From 6e8aa7d53e5251a87567a575cfca2ec349cfc486 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 21:28:43 +0100 Subject: [PATCH 12/30] feat: PolarsEngine Excel scan via calamine Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 60 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index c831dd3..45c04f3 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -41,7 +41,7 @@ impl PolarsEngine { .map_err(|e| read_err(path, e)), InputFormat::Parquet => LazyFrame::scan_parquet(path.into(), ScanArgsParquet::default()) .map_err(|e| read_err(path, e)), - _ => Err(read_err(path, "unsupported format (implemented in a later task)")), + InputFormat::Excel { sheet } => read_excel(path, sheet.as_deref()), } } @@ -164,6 +164,52 @@ where } } +fn read_excel(path: &str, sheet: Option<&str>) -> Result { + use calamine::{open_workbook_auto, Reader}; + + let mut workbook = open_workbook_auto(path).map_err(|e| read_err(path, e))?; + let sheet_name = match sheet { + Some(name) => name.to_string(), + None => workbook + .sheet_names() + .first() + .cloned() + .ok_or_else(|| read_err(path, "workbook has no sheets"))?, + }; + let range = workbook + .worksheet_range(&sheet_name) + .map_err(|e| read_err(path, e))?; + + let mut rows = range.rows(); + let headers: Vec = match rows.next() { + Some(first) => first.iter().map(cell_to_string).collect(), + None => return Ok(DataFrame::empty().lazy()), + }; + + let mut columns: Vec> = vec![Vec::new(); headers.len()]; + for row in rows { + for (idx, col) in columns.iter_mut().enumerate() { + let cell = row.get(idx).map(cell_to_string).unwrap_or_default(); + col.push(cell); + } + } + + let height = columns.first().map(|c| c.len()).unwrap_or(0); + let series: Vec = headers + .into_iter() + .zip(columns) + .map(|(name, values)| Series::new(name.into(), values).into_column()) + .collect(); + + DataFrame::new(height, series) + .map(|df| df.lazy()) + .map_err(|e| read_err(path, e)) +} + +fn cell_to_string(cell: &calamine::Data) -> String { + cell.to_string() +} + fn sql_err(sql: &str, source: PolarsError) -> DtooError { DtooError::Sql { context: "polars-sql".to_string(), @@ -184,6 +230,18 @@ mod tests { std::env::temp_dir().join(format!("dtoo-pe-{name}-{nanos}.{ext}")) } + #[test] + fn scan_excel_reads_existing_fixture() { + // Repo fixture from testdata/. Default (first) sheet. + let engine = PolarsEngine::new(); + let lf = engine + .scan("testdata/xlsxs/trips.xlsx", &InputFormat::Excel { sheet: None }) + .unwrap(); + let df = engine.collect(lf).unwrap(); + assert!(df.height() > 0); + assert!(df.width() > 0); + } + #[test] fn scan_ndjson_reads_rows() { let path = tmp("ndjson", "ndjson"); From f69b629ad11b989792415c21cdbe191dcdc334d3 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 22:39:09 +0100 Subject: [PATCH 13/30] test: make Excel scan test self-contained (testdata is gitignored) Replace the fixture-dependent `scan_excel_reads_existing_fixture` test with two self-contained tests that generate a tiny workbook at test time using `rust_xlsxwriter` (dev-dependency only). Removes the gitignored `testdata/xlsxs/trips.xlsx` dependency so the test suite passes on CI and fresh checkouts without manual fixture copying. Co-Authored-By: Claude Sonnet 4.6 --- Cargo.lock | 10 ++++++++++ Cargo.toml | 3 +++ src/polars_engine.rs | 46 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b156f3..5fc6656 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1070,6 +1070,7 @@ dependencies = [ "hex", "indicatif", "polars", + "rust_xlsxwriter", "serde", "serde_json", "serde_yaml", @@ -3501,6 +3502,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "rust_xlsxwriter" +version = "0.95.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f281b687352597d29efaad39701d1167d5c48aa76fb973e392bc13e9d44e7f36" +dependencies = [ + "zip 7.2.0", +] + [[package]] name = "rustc-hash" version = "2.1.2" diff --git a/Cargo.toml b/Cargo.toml index 3aeb921..ea993d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,3 +33,6 @@ lto = true codegen-units = 1 strip = true panic = "abort" + +[dev-dependencies] +rust_xlsxwriter = "0.95.0" diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 45c04f3..5fbd4ca 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -230,16 +230,52 @@ mod tests { std::env::temp_dir().join(format!("dtoo-pe-{name}-{nanos}.{ext}")) } + fn write_test_xlsx(path: &std::path::Path, sheet: &str) { + use rust_xlsxwriter::Workbook; + let mut workbook = Workbook::new(); + let worksheet = workbook.add_worksheet(); + worksheet.set_name(sheet).unwrap(); + worksheet.write_string(0, 0, "id").unwrap(); + worksheet.write_string(0, 1, "name").unwrap(); + worksheet.write_string(1, 0, "1").unwrap(); + worksheet.write_string(1, 1, "alice").unwrap(); + worksheet.write_string(2, 0, "2").unwrap(); + worksheet.write_string(2, 1, "bob").unwrap(); + workbook.save(path).unwrap(); + } + + #[test] + fn scan_excel_reads_default_sheet() { + let path = tmp("xlsx-default", "xlsx"); + write_test_xlsx(&path, "Sheet1"); + let engine = PolarsEngine::new(); + + let lf = engine + .scan(path.to_str().unwrap(), &InputFormat::Excel { sheet: None }) + .unwrap(); + let df = engine.collect(lf).unwrap(); + + assert_eq!(df.height(), 2); + assert_eq!(df.get_column_names(), vec!["id", "name"]); + let _ = std::fs::remove_file(path); + } + #[test] - fn scan_excel_reads_existing_fixture() { - // Repo fixture from testdata/. Default (first) sheet. + fn scan_excel_reads_named_sheet() { + let path = tmp("xlsx-named", "xlsx"); + write_test_xlsx(&path, "Data"); let engine = PolarsEngine::new(); + let lf = engine - .scan("testdata/xlsxs/trips.xlsx", &InputFormat::Excel { sheet: None }) + .scan( + path.to_str().unwrap(), + &InputFormat::Excel { sheet: Some("Data".to_string()) }, + ) .unwrap(); let df = engine.collect(lf).unwrap(); - assert!(df.height() > 0); - assert!(df.width() > 0); + + assert_eq!(df.height(), 2); + let _ = std::fs::remove_file(path); } #[test] From 2e85677e9d2aecc646237f38cf7ebe38801b8ee1 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 22:52:01 +0100 Subject: [PATCH 14/30] feat: PolarsEngine union-by-name concat (schema evolution) Co-Authored-By: Claude Sonnet 4.6 --- Cargo.toml | 2 +- src/polars_engine.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ea993d7..6fdfc21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ aes = "0.8" ecb = "0.1" sha1 = "0.10" base64 = "0.22" -polars = { version = "0.54.4", features = ["lazy", "sql", "csv", "parquet", "json", "strings", "dtype-full"] } +polars = { version = "0.54.4", features = ["lazy", "sql", "csv", "parquet", "json", "strings", "dtype-full", "diagonal_concat"] } calamine = "0.35.0" flate2 = "1.1.9" zstd = "0.13.3" diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 5fbd4ca..4c52c3f 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -50,6 +50,14 @@ impl PolarsEngine { lf.collect().map_err(|e| read_err("(query)", e)) } + /// Concatenate frames union-by-name (diagonal), filling missing columns with null. + pub fn concat_by_name(&self, frames: Vec) -> Result { + if frames.is_empty() { + return Err(read_err("(concat)", "no frames to concatenate")); + } + concat_lf_diagonal(frames, UnionArgs::default()).map_err(|e| read_err("(concat)", e)) + } + /// Count rows without retaining the materialized frame. pub fn row_count(&self, lf: LazyFrame) -> Result { Ok(self.collect(lf)?.height()) @@ -405,6 +413,28 @@ mod tests { let _ = std::fs::remove_file(dst); } + #[test] + fn concat_by_name_aligns_differing_columns() { + let engine = PolarsEngine::new(); + let a = df!["id" => [1i64], "name" => ["alice"]].unwrap().lazy(); + let b = df!["id" => [2i64], "extra" => ["x"]].unwrap().lazy(); + + let merged = engine.concat_by_name(vec![a, b]).unwrap(); + let df = engine.collect(merged).unwrap(); + + assert_eq!(df.height(), 2); + let mut names: Vec = df + .get_column_names() + .iter() + .map(|s| s.to_string()) + .collect(); + names.sort(); + assert_eq!( + names, + vec!["extra".to_string(), "id".to_string(), "name".to_string()] + ); + } + #[test] fn csv_zstd_output_is_decompressible() { let src = tmp("zs-src", "csv"); From 19b910c01a11425bc1b3423a2d85c09d5c9d69c5 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 22:56:07 +0100 Subject: [PATCH 15/30] feat: PolarsEngine SQL execution via SQLContext Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 4c52c3f..443484e 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -63,6 +63,22 @@ impl PolarsEngine { Ok(self.collect(lf)?.height()) } + /// Run user SQL with `base` registered as the magic table `_` and each ref + /// registered under its name. Returns the resulting LazyFrame. + pub fn run_sql( + &self, + base: LazyFrame, + refs: &[(String, LazyFrame)], + sql: &str, + ) -> Result { + let mut ctx = polars::sql::SQLContext::new(); + ctx.register("_", base); + for (name, lf) in refs { + ctx.register(name, lf.clone()); + } + ctx.execute(sql).map_err(|e| sql_err(sql, e)) + } + /// Write a DataFrame to a file path or, when `dest` is None, to stdout. pub fn write( &self, @@ -435,6 +451,38 @@ mod tests { ); } + #[test] + fn run_sql_filters_and_joins() { + let engine = PolarsEngine::new(); + let base = df!["id" => [1i64, 2], "region_id" => [10i64, 20]].unwrap().lazy(); + let regions = df!["id" => [10i64, 20], "region_name" => ["EMEA", "APAC"]].unwrap().lazy(); + + let out = engine + .run_sql( + base, + &[("regions".to_string(), regions)], + "SELECT _.id, r.region_name FROM _ JOIN regions r ON _.region_id = r.id WHERE _.id = 1", + ) + .unwrap(); + let df = engine.collect(out).unwrap(); + + assert_eq!(df.height(), 1); + assert_eq!(df.column("region_name").unwrap().str().unwrap().get(0), Some("EMEA")); + } + + #[test] + fn run_sql_returns_error_on_unsupported_sql_without_hanging() { + // In Polars 0.54, DELETE is actually implemented (inverted filter), so + // it silently succeeds. INSERT is unhandled and hits the catch-all + // `_ => polars_bail!` branch in SQLContext::execute, making it the + // canonical unsupported-statement test. The non-negotiable requirement + // is that an error is returned as a Result (no panic, no hang). + let engine = PolarsEngine::new(); + let base = df!["id" => [1i64]].unwrap().lazy(); + let result = engine.run_sql(base, &[], "INSERT INTO _ VALUES (2)"); + assert!(matches!(result, Err(DtooError::Sql { .. }))); + } + #[test] fn csv_zstd_output_is_decompressible() { let src = tmp("zs-src", "csv"); From c901e414ee041f51bb546cc7f5e9f6d6efdb7514 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 22:58:02 +0100 Subject: [PATCH 16/30] feat: PolarsEngine lazy schema introspection Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 443484e..fbc904d 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -79,6 +79,18 @@ impl PolarsEngine { ctx.execute(sql).map_err(|e| sql_err(sql, e)) } + /// Return (column name, dtype) pairs without materializing the frame. + pub fn schema_of(&self, lf: &LazyFrame) -> Result, DtooError> { + let schema = lf + .clone() + .collect_schema() + .map_err(|e| DtooError::Schema { message: e.to_string() })?; + Ok(schema + .iter() + .map(|(name, dtype)| (name.to_string(), dtype.clone())) + .collect()) + } + /// Write a DataFrame to a file path or, when `dest` is None, to stdout. pub fn write( &self, @@ -483,6 +495,17 @@ mod tests { assert!(matches!(result, Err(DtooError::Sql { .. }))); } + #[test] + fn schema_of_returns_names_and_types() { + let engine = PolarsEngine::new(); + let lf = df!["id" => [1i64], "name" => ["alice"]].unwrap().lazy(); + + let schema = engine.schema_of(&lf).unwrap(); + let names: Vec<&str> = schema.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, vec!["id", "name"]); + assert_eq!(schema[0].1, DataType::Int64); + } + #[test] fn csv_zstd_output_is_decompressible() { let src = tmp("zs-src", "csv"); From d1475d8cbc5500db68fef0632b8ff2f6a55ce67b Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 23:02:11 +0100 Subject: [PATCH 17/30] test: malformed input surfaces a clear error, never hangs Regression test proving ragged CSV rows produce Err(DtooError::FileProcess) rather than a silent Ok or hang. Also adds .with_ignore_errors(false) to the LazyCsvReader builder to make the strict-parsing contract explicit and durable. Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index fbc904d..c834425 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -34,6 +34,7 @@ impl PolarsEngine { InputFormat::Csv { delimiter } => LazyCsvReader::new(path.into()) .with_separator(*delimiter as u8) .with_has_header(true) + .with_ignore_errors(false) // explicit: ragged/unparseable rows must error, not silently drop .finish() .map_err(|e| read_err(path, e)), InputFormat::Ndjson => LazyJsonLineReader::new(path.into()) @@ -506,6 +507,30 @@ mod tests { assert_eq!(schema[0].1, DataType::Int64); } + #[test] + fn malformed_csv_surfaces_error_not_hang() { + // Regression for the motivating bug: the old DuckDB engine silently hung + // on a malformed CSV field, costing the maintainer half a day. + // + // This exercises ragged rows: the header declares 2 columns but one data + // row has 4 fields. `.with_ignore_errors(false)` on the LazyCsvReader + // makes this an explicit contract — malformed input must surface as + // Err(DtooError::FileProcess), never a silent Ok or a hang. + let path = tmp("bad", "csv"); + std::fs::write(&path, "id,name\n1,alice\n2,bob,extra,boom\n").unwrap(); + let engine = PolarsEngine::new(); + + let result = engine + .scan(path.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .and_then(|lf| engine.collect(lf)); + + assert!( + matches!(result, Err(DtooError::FileProcess { .. })), + "expected a FileProcess error for malformed CSV, got: {result:?}" + ); + let _ = std::fs::remove_file(path); + } + #[test] fn csv_zstd_output_is_decompressible() { let src = tmp("zs-src", "csv"); From 2685f085b51ed03e65c5e010f51060ab6ff75194 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 23:04:47 +0100 Subject: [PATCH 18/30] test: cloud paths rejected with clear deferred-feature error Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index c834425..9796d23 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -531,6 +531,20 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn cloud_paths_are_rejected_with_clear_error() { + let engine = PolarsEngine::new(); + let result = engine.scan("s3://bucket/data.parquet", &InputFormat::Parquet); + match result { + Err(DtooError::Config { message }) => { + assert!(message.contains("cloud storage")); + assert!(message.contains("not supported")); + } + Err(other) => panic!("expected Config error, got a different error: {other}"), + Ok(_) => panic!("expected Config error, got Ok"), + } + } + #[test] fn csv_zstd_output_is_decompressible() { let src = tmp("zs-src", "csv"); From 96f407bc69ae88cbef02ef83b2e92034571ac358 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 23:07:51 +0100 Subject: [PATCH 19/30] style: cargo fmt after Phase 1 --- src/config.rs | 4 +- src/polars_engine.rs | 129 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 105 insertions(+), 28 deletions(-) diff --git a/src/config.rs b/src/config.rs index 0c7d2d5..c69c875 100644 --- a/src/config.rs +++ b/src/config.rs @@ -151,7 +151,9 @@ fn validate_top_level_types(value: &serde_yaml::Value) -> Result<(), DtooError> "profile" => ensure_mapping_or_null(key, val)?, "cloud" => ensure_mapping_or_null(key, val)?, "crypto_profiles" - if !(val.is_null() || val.as_sequence().is_some() || val.as_mapping().is_some()) => + if !(val.is_null() + || val.as_sequence().is_some() + || val.as_mapping().is_some()) => { return Err(config_type_error(key, "list or mapping")); } diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 9796d23..068443c 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -40,8 +40,10 @@ impl PolarsEngine { InputFormat::Ndjson => LazyJsonLineReader::new(path.into()) .finish() .map_err(|e| read_err(path, e)), - InputFormat::Parquet => LazyFrame::scan_parquet(path.into(), ScanArgsParquet::default()) - .map_err(|e| read_err(path, e)), + InputFormat::Parquet => { + LazyFrame::scan_parquet(path.into(), ScanArgsParquet::default()) + .map_err(|e| read_err(path, e)) + } InputFormat::Excel { sheet } => read_excel(path, sheet.as_deref()), } } @@ -82,10 +84,9 @@ impl PolarsEngine { /// Return (column name, dtype) pairs without materializing the frame. pub fn schema_of(&self, lf: &LazyFrame) -> Result, DtooError> { - let schema = lf - .clone() - .collect_schema() - .map_err(|e| DtooError::Schema { message: e.to_string() })?; + let schema = lf.clone().collect_schema().map_err(|e| DtooError::Schema { + message: e.to_string(), + })?; Ok(schema .iter() .map(|(name, dtype)| (name.to_string(), dtype.clone())) @@ -186,8 +187,7 @@ where sink.flush().map_err(write_err) } Some(CompressionCodec::Gzip) => { - let mut encoder = - flate2::write::GzEncoder::new(sink, flate2::Compression::default()); + let mut encoder = flate2::write::GzEncoder::new(sink, flate2::Compression::default()); write_body(&mut encoder)?; encoder.finish().map_err(write_err)?; Ok(()) @@ -202,7 +202,7 @@ where } fn read_excel(path: &str, sheet: Option<&str>) -> Result { - use calamine::{open_workbook_auto, Reader}; + use calamine::{Reader, open_workbook_auto}; let mut workbook = open_workbook_auto(path).map_err(|e| read_err(path, e))?; let sheet_name = match sheet { @@ -306,7 +306,9 @@ mod tests { let lf = engine .scan( path.to_str().unwrap(), - &InputFormat::Excel { sheet: Some("Data".to_string()) }, + &InputFormat::Excel { + sheet: Some("Data".to_string()), + }, ) .unwrap(); let df = engine.collect(lf).unwrap(); @@ -318,10 +320,16 @@ mod tests { #[test] fn scan_ndjson_reads_rows() { let path = tmp("ndjson", "ndjson"); - std::fs::write(&path, "{\"id\":1,\"name\":\"alice\"}\n{\"id\":2,\"name\":\"bob\"}\n").unwrap(); + std::fs::write( + &path, + "{\"id\":1,\"name\":\"alice\"}\n{\"id\":2,\"name\":\"bob\"}\n", + ) + .unwrap(); let engine = PolarsEngine::new(); - let lf = engine.scan(path.to_str().unwrap(), &InputFormat::Ndjson).unwrap(); + let lf = engine + .scan(path.to_str().unwrap(), &InputFormat::Ndjson) + .unwrap(); assert_eq!(engine.row_count(lf).unwrap(), 2); let _ = std::fs::remove_file(path); } @@ -351,11 +359,26 @@ mod tests { let engine = PolarsEngine::new(); let df = engine - .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .collect( + engine + .scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(), + ) + .unwrap(); + engine + .write( + df, + Some(pq.as_path()), + ExportFormat::Parquet, + true, + ',', + None, + ) .unwrap(); - engine.write(df, Some(pq.as_path()), ExportFormat::Parquet, true, ',', None).unwrap(); - let back = engine.scan(pq.to_str().unwrap(), &InputFormat::Parquet).unwrap(); + let back = engine + .scan(pq.to_str().unwrap(), &InputFormat::Parquet) + .unwrap(); assert_eq!(engine.row_count(back).unwrap(), 2); let _ = std::fs::remove_file(src); let _ = std::fs::remove_file(pq); @@ -369,11 +392,26 @@ mod tests { let engine = PolarsEngine::new(); let df = engine - .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .collect( + engine + .scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(), + ) + .unwrap(); + engine + .write( + df, + Some(dst.as_path()), + ExportFormat::Ndjson, + true, + ',', + None, + ) .unwrap(); - engine.write(df, Some(dst.as_path()), ExportFormat::Ndjson, true, ',', None).unwrap(); - let back = engine.scan(dst.to_str().unwrap(), &InputFormat::Ndjson).unwrap(); + let back = engine + .scan(dst.to_str().unwrap(), &InputFormat::Ndjson) + .unwrap(); assert_eq!(engine.row_count(back).unwrap(), 1); let _ = std::fs::remove_file(src); let _ = std::fs::remove_file(dst); @@ -387,7 +425,11 @@ mod tests { let engine = PolarsEngine::new(); let df = engine - .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .collect( + engine + .scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(), + ) .unwrap(); engine .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', None) @@ -407,7 +449,11 @@ mod tests { let dst = tmp("noh-dst", "csv"); let engine = PolarsEngine::new(); let df = engine - .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .collect( + engine + .scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(), + ) .unwrap(); engine .write(df, Some(dst.as_path()), ExportFormat::Csv, false, ',', None) @@ -427,10 +473,21 @@ mod tests { let engine = PolarsEngine::new(); let df = engine - .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .collect( + engine + .scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(), + ) .unwrap(); engine - .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', Some(CompressionCodec::Gzip)) + .write( + df, + Some(dst.as_path()), + ExportFormat::Csv, + true, + ',', + Some(CompressionCodec::Gzip), + ) .unwrap(); let bytes = std::fs::read(&dst).unwrap(); @@ -467,8 +524,12 @@ mod tests { #[test] fn run_sql_filters_and_joins() { let engine = PolarsEngine::new(); - let base = df!["id" => [1i64, 2], "region_id" => [10i64, 20]].unwrap().lazy(); - let regions = df!["id" => [10i64, 20], "region_name" => ["EMEA", "APAC"]].unwrap().lazy(); + let base = df!["id" => [1i64, 2], "region_id" => [10i64, 20]] + .unwrap() + .lazy(); + let regions = df!["id" => [10i64, 20], "region_name" => ["EMEA", "APAC"]] + .unwrap() + .lazy(); let out = engine .run_sql( @@ -480,7 +541,10 @@ mod tests { let df = engine.collect(out).unwrap(); assert_eq!(df.height(), 1); - assert_eq!(df.column("region_name").unwrap().str().unwrap().get(0), Some("EMEA")); + assert_eq!( + df.column("region_name").unwrap().str().unwrap().get(0), + Some("EMEA") + ); } #[test] @@ -552,10 +616,21 @@ mod tests { let dst = tmp("zs-dst", "csv.zst"); let engine = PolarsEngine::new(); let df = engine - .collect(engine.scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }).unwrap()) + .collect( + engine + .scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(), + ) .unwrap(); engine - .write(df, Some(dst.as_path()), ExportFormat::Csv, true, ',', Some(CompressionCodec::Zstd)) + .write( + df, + Some(dst.as_path()), + ExportFormat::Csv, + true, + ',', + Some(CompressionCodec::Zstd), + ) .unwrap(); let bytes = std::fs::read(&dst).unwrap(); From 90cd7c257afd6e9e901eb412f23e0b5a388e27b2 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 23:13:59 +0100 Subject: [PATCH 20/30] harden: error on over-wide Excel rows; cover stdout/ndjson-gzip/missing-sheet paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calamine 0.35 pads all rows to the sheet's max width, so an over-wide data row silently extends the header row with Empty cells (empty strings) rather than raising an error. read_excel now detects trailing empty-string header columns — which only appear when data rows are wider than the user-declared headers — and returns DtooError::FileProcess instead of ingesting data under unnamed columns. Also adds four new tests: scan_excel_over_wide_row_errors (Fix 1), scan_excel_missing_named_sheet_errors, write_csv_to_stdout_sink_succeeds, and ndjson_gzip_output_is_decompressible. Suite grows from 16 to 20. Co-Authored-By: Claude Sonnet 4.6 --- src/polars_engine.rs | 160 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 159 insertions(+), 1 deletion(-) diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 068443c..dfb2bb1 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -218,13 +218,59 @@ fn read_excel(path: &str, sheet: Option<&str>) -> Result { .map_err(|e| read_err(path, e))?; let mut rows = range.rows(); + + // Calamine 0.35 pads every row to the sheet's maximum used width, so an + // over-wide data row causes calamine to also widen the header row — but + // the extra header cells are `Empty` (serialised to ""). We detect this + // by checking that every header cell has a non-empty name. A trailing + // empty-string header means a data row had more cells than the user + // declared columns, which is a data-integrity error: the value would + // otherwise be silently ingested under an unnamed column. let headers: Vec = match rows.next() { - Some(first) => first.iter().map(cell_to_string).collect(), + Some(first) => { + // Strip any trailing empty-string columns that calamine appended + // purely as padding (they have no header name). After stripping, + // if the rightmost *named* column is genuinely empty that is also + // an error — we only strip pure trailing padding. + let all: Vec = first.iter().map(cell_to_string).collect(); + // Find how many trailing columns are empty (calamine padding). + let trailing_empty = all.iter().rev().take_while(|s| s.is_empty()).count(); + let named_count = all.len() - trailing_empty; + if named_count == 0 { + return Ok(DataFrame::empty().lazy()); + } + // If any non-trailing header cell is empty the sheet is malformed. + for (i, h) in all[..named_count].iter().enumerate() { + if h.is_empty() { + return Err(read_err( + path, + format!("header column {i} has an empty name — sheet is malformed"), + )); + } + } + // If there were trailing-empty padding columns, that means at + // least one data row had more cells than declared headers. Error + // now rather than silently ingesting garbage columns. + if trailing_empty > 0 { + return Err(read_err( + path, + format!( + "at least one data row has {} extra cell(s) beyond the {} declared \ + header columns — refusing to silently drop data", + trailing_empty, named_count + ), + )); + } + all + } None => return Ok(DataFrame::empty().lazy()), }; let mut columns: Vec> = vec![Vec::new(); headers.len()]; for row in rows { + // Calamine pads all rows to the sheet width, so row.len() always + // equals headers.len() here. We iterate only up to headers.len() + // as a defensive measure in case of short rows (null-fill behaviour). for (idx, col) in columns.iter_mut().enumerate() { let cell = row.get(idx).map(cell_to_string).unwrap_or_default(); col.push(cell); @@ -609,6 +655,118 @@ mod tests { } } + // ----------------------------------------------------------------------- + // New hardening tests (Fix 1 + four coverage tests) + // ----------------------------------------------------------------------- + + /// Calamine 0.35 pads all rows to the sheet's maximum used width, so an + /// over-wide data row makes the header row also wider — but the extra + /// header cells are Empty (""). `read_excel` detects trailing empty + /// header columns and returns `FileProcess` rather than silently ingesting + /// data under unnamed columns. + #[test] + fn scan_excel_over_wide_row_errors() { + use rust_xlsxwriter::Workbook; + let path = tmp("xlsx-wide", "xlsx"); + let mut workbook = Workbook::new(); + let ws = workbook.add_worksheet(); + ws.write_string(0, 0, "id").unwrap(); + ws.write_string(0, 1, "name").unwrap(); + ws.write_string(1, 0, "1").unwrap(); + ws.write_string(1, 1, "alice").unwrap(); + ws.write_string(1, 2, "EXTRA").unwrap(); // third cell, no header + workbook.save(&path).unwrap(); + + let engine = PolarsEngine::new(); + let result = engine + .scan(path.to_str().unwrap(), &InputFormat::Excel { sheet: None }) + .and_then(|lf| engine.collect(lf)); + assert!( + matches!(result, Err(DtooError::FileProcess { .. })), + "expected FileProcess error for over-wide Excel row, got: {result:?}" + ); + let _ = std::fs::remove_file(path); + } + + /// Requesting a named sheet that does not exist must yield a Result error, + /// never a panic. Calamine 0.35 returns Err(WorksheetNotFound(…)) which + /// is already mapped via `.map_err(|e| read_err(path, e))`. + #[test] + fn scan_excel_missing_named_sheet_errors() { + let path = tmp("xlsx-missing-sheet", "xlsx"); + write_test_xlsx(&path, "Sheet1"); + let engine = PolarsEngine::new(); + let result = engine.scan( + path.to_str().unwrap(), + &InputFormat::Excel { + sheet: Some("DoesNotExist".to_string()), + }, + ); + // scan may error eagerly, or error on collect — either way it must be + // a Result error, never a panic. + let result = result.and_then(|lf| engine.collect(lf)); + assert!( + matches!(result, Err(DtooError::FileProcess { .. })), + "expected FileProcess error for missing sheet, got: {result:?}" + ); + let _ = std::fs::remove_file(path); + } + + /// The `open_sink(None)` → stdout branch was previously untested. + /// Writing CSV to stdout (dest = None) must complete without error. + #[test] + fn write_csv_to_stdout_sink_succeeds() { + let src = tmp("stdout-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let engine = PolarsEngine::new(); + let df = engine + .collect( + engine + .scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(), + ) + .unwrap(); + // dest = None writes to stdout; we only assert it completes without error. + engine + .write(df, None, ExportFormat::Csv, true, ',', None) + .unwrap(); + let _ = std::fs::remove_file(src); + } + + /// Only CSV compression was previously tested; NDJSON shares the + /// write_with_optional_compression path but was unverified. + #[test] + fn ndjson_gzip_output_is_decompressible() { + let src = tmp("ndgz-src", "csv"); + std::fs::write(&src, "id,name\n1,alice\n").unwrap(); + let dst = tmp("ndgz-dst", "ndjson.gz"); + let engine = PolarsEngine::new(); + let df = engine + .collect( + engine + .scan(src.to_str().unwrap(), &InputFormat::Csv { delimiter: ',' }) + .unwrap(), + ) + .unwrap(); + engine + .write( + df, + Some(dst.as_path()), + ExportFormat::Ndjson, + true, + ',', + Some(CompressionCodec::Gzip), + ) + .unwrap(); + let bytes = std::fs::read(&dst).unwrap(); + let mut decoder = flate2::read::GzDecoder::new(&bytes[..]); + let mut text = String::new(); + std::io::Read::read_to_string(&mut decoder, &mut text).unwrap(); + assert!(text.contains("alice")); + let _ = std::fs::remove_file(src); + let _ = std::fs::remove_file(dst); + } + #[test] fn csv_zstd_output_is_decompressible() { let src = tmp("zs-src", "csv"); From 465e296945b1f561a2860c9425fcc337ca7cf2e5 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 23:38:17 +0100 Subject: [PATCH 21/30] docs: add Phase 2 implementation plan for Polars engine cutover Slice-based plan to rewire query_pipeline + commands onto PolarsEngine, reimplement masking/lineage/schema/profiler/crypto as native Polars operations, then delete DuckDB. Existing behavioral tests are the regression safety net. Co-Authored-By: Claude Opus 4.7 --- .../plans/2026-06-06-polars-engine-cutover.md | 422 ++++++++++++++++++ 1 file changed, 422 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-06-polars-engine-cutover.md diff --git a/docs/superpowers/plans/2026-06-06-polars-engine-cutover.md b/docs/superpowers/plans/2026-06-06-polars-engine-cutover.md new file mode 100644 index 0000000..d5755c3 --- /dev/null +++ b/docs/superpowers/plans/2026-06-06-polars-engine-cutover.md @@ -0,0 +1,422 @@ +# Polars Engine Cutover — Implementation Plan (Phase 2 of 2) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Rewire dtoo's pipeline and commands onto the `PolarsEngine` built in Phase 1, reimplement the internal stages (masking, lineage, limit, schema coercion, profiling, crypto) as native Polars operations instead of generated SQL, then delete DuckDB. + +**Architecture:** The data carrier changes from "a mutable `temp_results` table inside `DuckDbEngine`" to "a Polars value flowing through the pipeline": scan each file → per-file filter via `run_sql` → accumulate with `concat_by_name` → (optional schema coercion) → collect to `DataFrame` → crypto decrypt → post-SQL → masking → lineage → limit → output/profile. Only the user-facing SQL (`--where`/`--filter-sql`/`--post-sql`) stays SQL (Polars `SQLContext`); every internal stage becomes a typed Rust/Polars transform. + +**Tech Stack:** Rust, the Phase-1 `PolarsEngine` (Polars 0.54.x), `sha2` (masking/lineage hash — already a dep), existing pure-Rust crypto (`aes`/`ecb`/`aes-gcm`/`sha1`/`base64`). + +**Spec:** `docs/specs/34-polars-engine.md`. **Builds on:** `docs/superpowers/plans/2026-06-06-polars-engine-core.md` (merged). + +--- + +## Realized Phase-1 `PolarsEngine` API (target for all rewiring) + +```rust +impl PolarsEngine { + pub fn new() -> Self; // also: Default + pub fn scan(&self, path: &str, format: &InputFormat) -> Result; + pub fn collect(&self, lf: LazyFrame) -> Result; + pub fn row_count(&self, lf: LazyFrame) -> Result; + pub fn write(&self, df: DataFrame, dest: Option<&Path>, format: ExportFormat, + header: bool, delimiter: char, compression: Option) -> Result<(), DtooError>; + pub fn concat_by_name(&self, frames: Vec) -> Result; + pub fn run_sql(&self, base: LazyFrame, refs: &[(String, LazyFrame)], sql: &str) -> Result; + pub fn schema_of(&self, lf: &LazyFrame) -> Result, DtooError>; +} +``` +`run_sql` registers `base` as the magic table `_` and each ref by name. Cloud paths are rejected by `scan` with a clear `DtooError::Config`. + +## New pipeline data flow (replaces the DuckDB `temp_results` model) + +``` +for each resolved file: + lf = engine.scan(path, format) + lf = per_file_filter(engine, lf, refs, --where, --filter-sql) // run_sql; both => where then filter-sql + if origin lineage: lf = lf.with_column(lit(path).alias("_origin_file")) + frames.push(lf) +acc = engine.concat_by_name(frames) // union-by-name (schema evolution) +if --schema: acc = coerce_to_schema(acc, &columns) // project + cast; missing->null +df = engine.collect(acc) +df = crypto::decrypt(df, &profile) // before post-sql (matches current order) +if --post-sql: df = engine.collect(engine.run_sql(df.lazy(), refs, post_sql)) +df = mask_columns(df, &cols, salt) +df = apply_lineage(df, &lineage_manager) // batch_id/record_id/ts/hash + rename _origin_file +if --limit n: df = df.head(Some(n)) +row_count = df.height() +... expect-at-least / count ... +df = crypto::encrypt(df, &profile, &cols) // output encryption +engine.write(df, output, format, header, delimiter, compression) +if --profile: Profiler::generate(&df, &options) +``` + +## Critical safety net + +The existing tests are the regression guard and **mostly do not need rewriting**: +- `tests/cli_integration.rs` (3 tests) drive the real binary — engine-agnostic. +- `query_pipeline::tests` (~12 tests) call `QueryPipeline::run(&QueryArgs)` — engine-agnostic; they must keep passing unchanged. +- `masking`/`lineage`/`schema`/`profiler`/`reference_tables`/`crypto` inline tests construct a `DuckDbEngine` today; those get **ported** to the new data model in their respective tasks (the asserted behavior stays identical). + +Run `cargo test` after every task. Never delete DuckDB until the final teardown slice; until then both engines compile side by side. + +--- + +# Slice A — Decouple shared types + +### Task 1: Move shared enums out of the DuckDB engine module + +**Files:** +- Create: `src/types.rs` +- Modify: `src/engine.rs` (remove the moved definitions, import them), `src/main.rs` (add `mod types;`), and every file importing these from `crate::engine` (`query_pipeline.rs`, `polars_engine.rs`, `reference_tables.rs`, `schema.rs`, `output_writer.rs`, `inspect.rs`, `convert_command.rs`, `cli.rs` mappings as applicable). + +- [ ] **Step 1: Create `src/types.rs`** containing the engine-agnostic data types currently defined in `engine.rs`. Move (cut) these exact definitions verbatim from `engine.rs`: `InputFormat`, `ExportFormat`, `CompressionCodec`, `SchemaColumn`. Add `mod types;` to `main.rs`. + +- [ ] **Step 2: Re-point imports.** In `engine.rs` and `polars_engine.rs`, change `use crate::engine::{...}` / local definitions to `use crate::types::{InputFormat, ExportFormat, CompressionCodec, SchemaColumn};`. Grep for every `crate::engine::{`/`crate::engine::InputFormat` etc. and repoint to `crate::types`: + Run: `grep -rn "engine::\(InputFormat\|ExportFormat\|CompressionCodec\|SchemaColumn\)" src/` + +- [ ] **Step 3: Build + test.** Run: `cargo build && cargo test 2>&1 | grep "test result"`. Expected: all pass (146). No behavior change — pure move. + +- [ ] **Step 4: Commit.** +```bash +git add src/types.rs src/engine.rs src/polars_engine.rs src/main.rs src/*.rs +git commit -m "refactor: move shared format/schema types to types.rs (decouple from DuckDB engine)" +``` + +--- + +# Slice B — Reimplement internal stages as Polars (alongside DuckDB) + +These add NEW pure-Polars functions next to the existing DuckDB ones. Nothing is wired into the pipeline yet; the build stays green. + +### Task 2: Masking as a DataFrame transform + +**Files:** Modify `src/masking.rs`. + +Current behavior (preserve exactly): for each selected column, replace each non-null value `v` with `hex(sha256("{salt}:{column}:" || v_as_text))`; NULLs stay NULL; unknown column → `DtooError::Config` listing available columns. (The DuckDB impl used `sha256(...)` which returns lowercase hex.) + +- [ ] **Step 1: Write the failing test** (new, DataFrame-based) in `masking.rs` tests: +```rust + #[test] + fn mask_columns_is_deterministic_and_preserves_null() { + use polars::prelude::*; + let df = df![ + "email" => [Some("a@example.com"), Some("a@example.com"), None] + ].unwrap(); + let out = mask_dataframe(df, &["email".to_string()], "project-x").unwrap(); + let col = out.column("email").unwrap().str().unwrap(); + assert_eq!(col.get(0), col.get(1)); // deterministic + assert!(col.get(2).is_none()); // null preserved + assert_ne!(col.get(0), Some("a@example.com")); // actually hashed + } + + #[test] + fn mask_dataframe_unknown_column_errors() { + use polars::prelude::*; + let df = df!["email" => ["x"]].unwrap(); + let err = mask_dataframe(df, &["missing".to_string()], "").unwrap_err(); + assert!(matches!(err, DtooError::Config { .. })); + } +``` + +- [ ] **Step 2: Run → fail.** `cargo test masking::tests::mask_columns_is_deterministic` → FAIL (`mask_dataframe` undefined). + +- [ ] **Step 3: Implement `mask_dataframe`.** Add to `masking.rs` (keep the existing `MaskingEngine`/`parse_columns` for now — removed in teardown): +```rust +use polars::prelude::*; +use sha2::{Digest, Sha256}; + +/// Replace each selected column's non-null values with hex(sha256("{salt}:{col}:" + value)). +pub fn mask_dataframe( + mut df: DataFrame, + columns: &[String], + salt: &str, +) -> Result { + if columns.is_empty() { + return Ok(df); + } + let available: std::collections::HashSet = + df.get_column_names().iter().map(|s| s.to_string()).collect(); + for column in columns { + if !available.contains(column) { + let mut sorted: Vec = available.iter().cloned().collect(); + sorted.sort(); + return Err(DtooError::Config { + message: format!( + "mask column `{column}` not found. available columns: {}", + sorted.join(", ") + ), + }); + } + let prefix = format!("{salt}:{column}:"); + // Stringify the column, then map each non-null value to its salted sha256 hex. + let s = df.column(column).map_err(|e| DtooError::Config { message: e.to_string() })?; + let as_str = s.cast(&DataType::String).map_err(|e| DtooError::Config { message: e.to_string() })?; + let chunked = as_str.str().map_err(|e| DtooError::Config { message: e.to_string() })?; + let masked: StringChunked = chunked + .into_iter() + .map(|opt| opt.map(|v| { + let mut h = Sha256::new(); + h.update(prefix.as_bytes()); + h.update(v.as_bytes()); + hex::encode(h.finalize()) + })) + .collect(); + df.replace(column, masked.into_series()) + .map_err(|e| DtooError::Config { message: e.to_string() })?; + } + Ok(df) +} +``` +NOTE: adapt `StringChunked`/`.str()`/`.replace`/`into_series` to the exact Polars 0.54 names if they differ (the Phase-1 module already used `.str()` accessors successfully). The non-negotiable contract is the two tests above. + +- [ ] **Step 4: Run → pass.** `cargo test masking::tests::mask_` → PASS. Then `cargo test 2>&1 | grep "test result"` (all still pass). + +- [ ] **Step 5: Commit.** `git add src/masking.rs && git commit -m "feat: masking as native Polars DataFrame transform"` + +### Task 3: Lineage as DataFrame transforms + +**Files:** Modify `src/lineage.rs`. + +Preserve: `batch_id` (constant string col), `record_id` (per-row UUID v4 string), `batch_timestamp` (constant — emit as the rfc3339 string the run computed), `batch_hash` (constant string), `origin_file` (per-file; tagged during accumulation as `_origin_file`, then renamed). Keep `LineageManager::new`/`batch_id`/`batch_hash`/`batch_timestamp`/`compute_batch_hash`/`requires_origin_tracking` unchanged. Replace the engine-based `tag_rows_with_origin`/`apply_columns` with DataFrame-based equivalents. + +- [ ] **Step 1: Write the failing test:** +```rust + #[test] + fn apply_lineage_adds_requested_columns_and_renames_origin() { + use polars::prelude::*; + let df = df!["id" => [1i64], "_origin_file" => ["/tmp/a.csv"]].unwrap(); + let mgr = LineageManager::new( + Some("batch_id,record_id,origin_file"), + LineageContext { files: vec!["/tmp/a.csv".to_string()], ..LineageContext::default() }, + ).unwrap(); + let out = mgr.apply_to_dataframe(df).unwrap(); + let names = out.get_column_names(); + assert!(names.iter().any(|n| *n == "batch_id")); + assert!(names.iter().any(|n| *n == "record_id")); + assert!(names.iter().any(|n| *n == "origin_file")); + assert!(!names.iter().any(|n| *n == "_origin_file")); + assert_eq!(out.column("origin_file").unwrap().str().unwrap().get(0), Some("/tmp/a.csv")); + } +``` + +- [ ] **Step 2: Run → fail.** + +- [ ] **Step 3: Implement** `apply_to_dataframe(&self, df: DataFrame) -> Result` on `LineageManager` using `with_column`/`lit` for the constant columns, a generated `Vec` of UUIDs (length `df.height()`) for `record_id`, and rename/drop logic for `_origin_file` mirroring the existing `apply_columns` (error `DtooError::Schema` if `origin_file` requested but `_origin_file` absent; drop `_origin_file` if present but not requested). Example for record_id: +```rust + if self.requested.contains(&LineageColumn::RecordId) { + let ids: Vec = (0..df.height()).map(|_| Uuid::new_v4().to_string()).collect(); + df.with_column(Series::new("record_id".into(), ids))?; + } +``` +Use `polars::prelude::*` and map Polars errors to `DtooError::Schema { message }`. Keep `batch_timestamp` as its rfc3339 string (documented simplification vs DuckDB TIMESTAMP type). + +- [ ] **Step 4: Run → pass**, then full suite green. + +- [ ] **Step 5: Commit.** `git commit -m "feat: lineage columns as native Polars DataFrame transforms"` + +### Task 4: Schema coercion + DuckDB-type→Polars-dtype map + +**Files:** Modify `src/schema.rs`. + +Preserve: explicit schema projects to the declared columns in order, missing source columns become NULL, extra source columns are dropped, and each column is cast to the declared type. Unknown/invalid type → error (current test `invalid_duckdb_type_fails_at_table_creation` expects an error — now `DtooError::Schema` instead of `Sql`). + +- [ ] **Step 1: Write failing tests:** +```rust + #[test] + fn duckdb_type_maps_to_polars() { + assert_eq!(duckdb_type_to_polars("INTEGER").unwrap(), polars::prelude::DataType::Int32); + assert_eq!(duckdb_type_to_polars("BIGINT").unwrap(), polars::prelude::DataType::Int64); + assert_eq!(duckdb_type_to_polars("VARCHAR").unwrap(), polars::prelude::DataType::String); + assert!(duckdb_type_to_polars("NOPE_TYPE").is_err()); + } + + #[test] + fn coerce_projects_casts_and_nulls_missing() { + use polars::prelude::*; + let lf = df!["id" => ["1"], "extra" => ["x"]].unwrap().lazy(); + let cols = vec![ + SchemaColumn { name: "id".into(), data_type: "INTEGER".into() }, + SchemaColumn { name: "name".into(), data_type: "VARCHAR".into() }, + ]; + let out = coerce_to_schema(lf, &cols).unwrap().collect().unwrap(); + assert_eq!(out.get_column_names(), vec!["id", "name"]); // declared order, extra dropped + assert_eq!(out.column("id").unwrap().dtype(), &DataType::Int32); + assert!(out.column("name").unwrap().null_count() == 1); // missing -> null + } +``` + +- [ ] **Step 2: Run → fail.** + +- [ ] **Step 3: Implement** `duckdb_type_to_polars(&str) -> Result` and `coerce_to_schema(lf: LazyFrame, columns: &[SchemaColumn]) -> Result`. Type map (case-insensitive, trim params): `INTEGER|INT|INT4 → Int32`, `BIGINT|INT8|LONG → Int64`, `SMALLINT|INT2 → Int16`, `TINYINT → Int8`, `DOUBLE|FLOAT8 → Float64`, `REAL|FLOAT|FLOAT4 → Float32`, `BOOLEAN|BOOL → Boolean`, `VARCHAR|TEXT|STRING|CHAR → String`, `DATE → Date`, `TIMESTAMP|DATETIME → Datetime(TimeUnit::Microseconds, None)`, `DECIMAL(p,s)|NUMERIC → Decimal(Some(p), Some(s))` (parse the params; default if absent). Unknown → `Err(DtooError::Schema { message: format!("unsupported schema type `{t}`") })`. Build the projection with `select`: +```rust + let exprs: Vec = columns.iter().map(|c| { + let dt = duckdb_type_to_polars(&c.data_type)?; + // missing column -> typed null literal; present -> cast + Ok(if present.contains(&c.name) { + col(c.name.as_str()).cast(dt).alias(c.name.as_str()) + } else { + lit(NULL).cast(dt).alias(c.name.as_str()) + }) + }).collect::>()?; + Ok(lf.select(exprs)) +``` +(`present` = the LazyFrame's column names via `engine.schema_of` or `lf.clone().collect_schema()`.) + +- [ ] **Step 4: Run → pass**, then full suite green. + +- [ ] **Step 5: Commit.** `git commit -m "feat: explicit-schema coercion as Polars cast/project with type map"` + +### Task 5: Reference-table loading as LazyFrames + +**Files:** Modify `src/reference_tables.rs`. + +- [ ] **Step 1: Write failing test** that loads a CSV ref via the new function and asserts it yields a `(name, LazyFrame)` whose collected height matches. Use a temp CSV. + +- [ ] **Step 2: Run → fail.** + +- [ ] **Step 3: Implement** `load_reference_lazyframes(engine: &PolarsEngine, refs: &[ReferenceTable]) -> Result, DtooError>` — for each ref, `engine.scan(&spec.path, &spec.format)` and collect the `(name, lf)` pair. Keep `parse_reference_tables` unchanged (it already produces `ReferenceTable { name, path, format }`). Map a `LoadedReferenceTable { name, path, row_count }` too if the verbose logger still needs counts (compute via `engine.row_count(lf.clone())`). + +- [ ] **Step 4: Run → pass**, full suite green. + +- [ ] **Step 5: Commit.** `git commit -m "feat: reference tables loaded as Polars LazyFrames"` + +--- + +# Slice C — Port profiler and crypto (behavior-preserving) + +### Task 6: Profiler over a DataFrame + +**Files:** Modify `src/profiler.rs`. + +Keep `ColumnProfile`/`ProfileReport`/`ValueFrequency`/`ProfileOptions` and ALL rendering (`render_csv`, `render_html`, `write_report`, `HTML_TEMPLATE`, escapes) **unchanged**. Replace only the data-gathering (`Profiler::generate` body, `build_report`, `profile_column`, `top_values`, `text_patterns`) to compute from a `&DataFrame` via Polars expressions. Metric mapping (all values stringified to match current output; `None` when not applicable / when DuckDB returned NULL): +- `count` = `df.height()` (or non-null? current `count` = total rows COUNT(*)), `null_count` = `col.null_count()`, `null_percentage` = `round(100*null/count, 2)`, `distinct_count` = `col.n_unique()`. +- numeric cols: `min`/`max`/`mean`/`std`/`median`/`quantile(0.25)`/`quantile(0.75)` via Series aggregations. +- text (Utf8) cols: min/max/avg of `str.len_chars()` over non-null; plus `text_patterns` (replace digits→`d`, letters→`a`, then runs of `d`→`N`) via `str.replace_all` regex then `value_counts` top-5. +- date/time cols: min/max. +- all cols: `top_5_values` via `value_counts` sorted desc, limit 5, value stringified. +- `data_type` string: derive from the Polars dtype (e.g. `format!("{dtype}")` or a small mapping); `is_numeric`/`is_text`/`is_date_like` can switch on `DataType` instead of the type string. + +- [ ] **Step 1:** Change `Profiler::generate` signature to `generate(df: &DataFrame, options: &ProfileOptions)`. Port the two existing tests (`generates_json_profile_file`, `generates_html_profile_file`) to build a `DataFrame` with `df!` instead of a `DuckDbEngine`/`temp_results`, keeping the same assertions (`"row_count": 2`, ``, `sortTable`, `Toggle`). Keep `csv_renderer_escapes_commas_and_quotes` as-is (it doesn't touch the engine). +- [ ] **Step 2: Run → fail** (signature/body changed). +- [ ] **Step 3: Implement** the Polars-based gathering per the mapping above. Sampling (`--profile-sample < 100`): take a random fraction via `df.sample_frac` (or `df.head` of `n*pct` if a deterministic subset is acceptable — match "USING SAMPLE %" loosely; document the slight semantic difference). Keep the `sample_percentage` validation (1..=100). +- [ ] **Step 4: Run → pass**; full suite green (note: `query_pipeline::tests::pipeline_writes_profile_report` still calls through `QueryPipeline::run` — it will only pass once Slice D wires the new profiler; until then it may still use the old path. Keep the OLD `Profiler` path callable until Task 9 — see note). + +> IMPORTANT ordering: to keep the suite green before Slice D, do NOT delete the old engine-based code paths that `query_pipeline.rs` still calls. Add the new DataFrame-based `generate` as the primary, and if a name clash occurs, temporarily name the new one `generate_df` and switch `query_pipeline` to it in Task 9, removing the old in teardown. Choose whichever keeps `cargo test` green; report which you did. + +- [ ] **Step 5: Commit.** `git commit -m "feat: profiler computes statistics from a Polars DataFrame"` + +### Task 7: Crypto decrypt/encrypt over a DataFrame + +**Files:** Modify `src/crypto.rs`. **Read the full current file first** — the scheme logic (resolve_profile, detection, AES-128-ECB-SHA1 / AES-256-GCM-base64 encode/decode, failure modes) is pure Rust and must be **preserved unchanged**. Only the data access changes. + +Current engine-coupled functions to port: `discover_wrapped_values(engine, detection, cols)`, `decrypt_temp_results(engine, profile)`, `encrypt_columns(engine, profile, cols)`. They currently: list string columns via `information_schema`, read distinct non-null values via `SELECT DISTINCT col`, and rewrite values via `UPDATE temp_results SET col = ... WHERE col = ...`. + +- [ ] **Step 1: Write/port tests** to operate on a `DataFrame` (build with `df!`, columns of wrapped/plaintext strings) asserting the same decrypt/encrypt/discovery behavior the current tests assert. (Read the existing `#[cfg(test)]` block and translate each.) +- [ ] **Step 2: Run → fail.** +- [ ] **Step 3: Implement** DataFrame-based variants: + - `discover_wrapped_values(df: &DataFrame, detection, cols) -> Result, DtooError>` — iterate the relevant String columns' values, count total/encrypted per the detection markers. + - `decrypt_dataframe(df: DataFrame, profile) -> Result<(DataFrame, CryptoProcessResult), DtooError>` — for each in-scope String column, map each value through the existing decrypt routine (preserve failure_mode best-effort vs strict), replacing the column; track `decrypted_columns`. + - `encrypt_dataframe(df: DataFrame, profile, cols) -> Result` — symmetric. + Reuse the existing pure-Rust scheme functions verbatim; only swap the per-value read/write to Polars `StringChunked` map + `df.replace`. Keep `enforce_output_safety` as-is. +- [ ] **Step 4: Run → pass**; full suite green (old engine-based crypto fns remain until teardown; add new ones alongside, switch in Task 9). +- [ ] **Step 5: Commit.** `git commit -m "feat: crypto decrypt/encrypt operate on Polars DataFrames"` + +--- + +# Slice D — Integrate the pipeline and commands + +### Task 8: Output writer takes a DataFrame + +**Files:** Modify `src/output_writer.rs`. + +- [ ] **Step 1:** Change `OutputWriter::write`/`write_and_get_destination` to accept the accumulated `DataFrame` (and a `&PolarsEngine`) and call `engine.write(df, output, format, header, delimiter, compression)`. Keep `OutputWriterConfig` and the stdout/destination resolution. Port its tests to construct a `DataFrame`. +- [ ] **Step 2–4:** TDD; full suite green. +- [ ] **Step 5: Commit.** `git commit -m "feat: output writer emits a Polars DataFrame via PolarsEngine"` + +### Task 9: Rewrite the query pipeline onto PolarsEngine + +**Files:** Modify `src/query_pipeline.rs` (the big integration). **Do not change `QueryArgs` or the manifest/summary structs.** The ~12 `query_pipeline::tests` and the 3 integration tests are the contract — they must pass unchanged. + +- [ ] **Step 1:** Replace `DuckDbEngine` usage with `PolarsEngine` and rewrite the execute flow to the data-flow described at the top of this plan: + - `initialize_engine` → `PolarsEngine::new()` (drop cloud/excel-extension flags; cloud now errors via `scan`'s guard). + - `load_reference_tables` → `reference_tables::load_reference_lazyframes` → keep `Vec<(String, LazyFrame)>` for `run_sql`. + - `process_files` → for each file: `scan` → `per_file_filter` (helper below) → optional `_origin_file` literal → push to `frames`; track per-file row counts (collect each filtered lf's height for the manifest/verbose — acceptable to `row_count` per file) and on-error skip/fail semantics exactly as today. + - After loop: `concat_by_name(frames)` → if `--schema` `coerce_to_schema` → `collect` → `df`. + - `crypto` decrypt (Task 7 fn) → post-sql via `run_sql(df.lazy(), refs, post_sql)` then collect → `mask_dataframe` → `lineage.apply_to_dataframe` → `--limit` `df.head(Some(n))`. + - `row_count = df.height()`; expect-at-least, `--count`, output crypto encrypt, `output_writer.write(df,...)`, profiler (`Profiler::generate(&df, ...)`), fingerprint, manifest — all unchanged in semantics. + - `per_file_filter(engine, lf, refs, where, filter_sql)`: `(None,None)=>lf`; `(Some(w),None)=>run_sql(lf,&[],"SELECT * FROM _ WHERE {w}")`; `(None,Some(f))=>run_sql(lf,refs,f)`; `(Some(w),Some(f))=>run_sql(run_sql(lf,&[],where_q)?, refs, f)`. +- [ ] **Step 2: Run the pipeline tests** `cargo test query_pipeline 2>&1 | grep "test result"` and `cargo test --test cli_integration`. Iterate until all pass. These encode behavior (reference join, where-before-filter, lineage columns, masking determinism, limit, profile report, manifest on success/failure, count exit codes, on-error skip → exit 3). Fix until green. +- [ ] **Step 3: Full suite** `cargo test` green. +- [ ] **Step 4: Commit.** `git commit -m "feat: query pipeline runs on PolarsEngine end-to-end"` + +### Task 10: Rewrite `inspect` and `profile` commands + +**Files:** Modify `src/inspect.rs`, `src/profile_command.rs`. + +- [ ] **Step 1 (inspect):** replace `DuckDbEngine` with `PolarsEngine`: `scan` the file → `schema_of` for the schema table, `row_count` for count, and `collect` + `head(rows)` for the preview (render via the existing `comfy-table` formatting). Keep the unknown-extension error path (integration test `inspect_unknown_extension_returns_error_exit_code` must stay green). If `--crypto-discover`, call the Task-7 `discover_wrapped_values` on the collected DataFrame. +- [ ] **Step 2 (profile):** `scan` the single file → `collect` → `Profiler::generate(&df, &options)`; apply crypto decrypt first if `--crypto-profile`. Port its tests. +- [ ] **Step 3:** TDD/iterate; full suite green. +- [ ] **Step 4: Commit.** `git commit -m "feat: inspect and profile commands run on PolarsEngine"` + +### Task 11: De-DuckDB file resolution and fingerprint (cloud deferral) + +**Files:** Modify `src/file_resolution.rs`, `src/fingerprint.rs`. + +- [ ] **Step 1 (file_resolution):** remove the `resolve_cloud_glob` DuckDB path. Local globs already work without DuckDB (use the new native expansion: Polars scanners glob natively, but file_resolution needs a concrete list for per-file processing/lineage — use the `glob` crate is being removed, so expand with `std`/`globset`? **Decision:** keep local glob expansion using the existing non-DuckDB code path if present; if the only globber was DuckDB, add a tiny local glob via the `glob` crate and DEFER removing `glob` to teardown only if still used). Cloud glob(`s3://` etc.) → return `DtooError::Config { "cloud storage ... not supported in this build yet" }` (consistent with `reject_cloud`). +- [ ] **Step 2 (fingerprint):** remove the DuckDB `read_blob` cloud path; cloud paths → the same deferred-cloud `Config` error. Local fingerprint (sha256 of file bytes) is already pure Rust — keep it. +- [ ] **Step 3:** TDD/iterate; full suite green. (If `glob` is still needed for local expansion, keep it — adjust the teardown task accordingly.) +- [ ] **Step 4: Commit.** `git commit -m "feat: defer cloud in file resolution and fingerprint with clear errors"` + +--- + +# Slice E — Teardown + +### Task 12: Delete DuckDB and dead code + +**Files:** Delete `src/engine.rs`; modify `src/main.rs`, `Cargo.toml`, and any remaining importers. + +- [ ] **Step 1:** Remove `mod engine;` from `main.rs` and delete `src/engine.rs`. Fix every remaining `crate::engine::` import (there should be none after Slices A–D except the deleted DuckDB types — they now live in `types.rs`). +- [ ] **Step 2:** Remove the old engine-coupled functions made dead in Slice B/C: `MaskingEngine` + `available_columns` (if unused), `LineageManager::tag_rows_with_origin`/`apply_columns` (the engine versions), `SchemaManager::initialize_temp_results`/`insert_file_rows` (engine versions) and `projected_query_for_schema`/`source_column_lookup`, the old `Profiler` engine path, the old crypto engine fns. Grep to confirm no callers: `grep -rn "DuckDbEngine\|temp_results\|\.execute(\|query_arrow" src/` → expect zero. +- [ ] **Step 3:** Remove now-unused `sql_utils.rs` (`escape_sql_literal`/`quote_identifier`) if nothing references it (`grep -rn "sql_utils\|escape_sql_literal\|quote_identifier" src/`); remove `mod sql_utils;`. +- [ ] **Step 4:** `cargo remove duckdb` and `cargo remove glob` (only if Task 11 confirmed glob is unused). Remove the `#![allow(dead_code)]` on `mod polars_engine;` in `main.rs` and `#![allow(unused_imports)]` in `polars_engine.rs`; fix any genuine dead-code/unused-import warnings that surface (e.g. the now-used `sql_err`). +- [ ] **Step 5:** `cargo build && cargo test && cargo clippy --all-targets -- -D warnings && cargo fmt --check`. All green/clean. +- [ ] **Step 6: Commit.** `git commit -m "refactor: remove DuckDB engine, dependency, and dead SQL helpers"` + +### Task 13: Documentation + +**Files:** Modify `docs/DESIGN.md`, `docs/USER_GUIDE.md`; mark `docs/specs/03-duckdb-engine.md` superseded. + +- [ ] **Step 1:** Update DESIGN.md Core Engine section to Polars; record the SQL surface limitations (Polars SQL: `DELETE`/`UPDATE` act as transforms rather than errors; window functions buggy; narrower function library) and the cloud-deferred status. Note Excel is read eagerly as all-String (type inference deferred) and over-wide Excel rows error. +- [ ] **Step 2:** Add a one-line banner to `docs/specs/03-duckdb-engine.md`: "Superseded by docs/specs/34-polars-engine.md (DuckDB→Polars conversion)." +- [ ] **Step 3:** Update USER_GUIDE.md anywhere it claims runtime cloud/Excel-extension behavior. +- [ ] **Step 4: Commit.** `git commit -m "docs: update for Polars engine, SQL limitations, and cloud deferral"` + +### Task 14: Final Phase-2 gate + +- [ ] **Step 1:** `cargo test` — all pass (the original 126 behavioral tests + Phase-1 engine tests + any ported stage tests). +- [ ] **Step 2:** `cargo clippy --all-targets -- -D warnings` — clean. +- [ ] **Step 3:** `cargo fmt --check` — clean (else `cargo fmt` + commit). +- [ ] **Step 4:** Sanity-run the real binary on a small CSV: `echo "id,amt\n1,100\n2,200" > /tmp/t.csv && cargo run -- query /tmp/t.csv --where "amt > 100"` → returns the filtered row; confirms the end-to-end Polars path. Also confirm a cloud path errors clearly: `cargo run -- query s3://x/y.csv` → clear "not supported" message, no hang. +- [ ] **Step 5:** Confirm DuckDB is gone: `grep -rn duckdb src/ Cargo.toml` → zero matches. + +--- + +## Self-Review Notes (coverage against spec 34 + Phase-1 handoff) + +- Stage map (masking/lineage/limit/crypto/profiling → native Polars) — Tasks 2,3,6,7,9. ✅ +- User SQL stays on `SQLContext`; `_` + refs — Task 9 `per_file_filter` + post-sql. ✅ +- Schema evolution (`concat_by_name`) + explicit `--schema` coercion + type map — Tasks 9, 4. ✅ +- Reference tables as LazyFrames — Task 5. ✅ +- Commands (query/convert/inspect/profile/fingerprint) on Polars — Tasks 9,10,11 (convert delegates to the pipeline; verify in Task 9). ✅ +- Cloud deferral with clear errors — Task 11 (+ Phase-1 `reject_cloud`). ✅ +- Remove `duckdb`/`glob`/dead SQL helpers; drop temporary allows — Task 12. ✅ +- Docs incl. SQL limitations (DELETE/UPDATE/window), Excel notes — Task 13. ✅ +- Regression safety: existing `query_pipeline`/integration tests kept and must pass unchanged — Task 9. ✅ + +## Risks / watch-items +- **`batch_timestamp`** emitted as rfc3339 string (not a DuckDB TIMESTAMP type) — verify the lineage test only checks header presence (it does). +- **Profiler sampling** semantics differ slightly (`sample_frac` vs DuckDB `USING SAMPLE`) — acceptable; document. +- **`glob` removal** is conditional on Task 11 confirming local expansion no longer needs it. +- **Polars SQL `DELETE`/`UPDATE`** silently transform rather than error — documented (Task 13), not fixed (out of scope). From 8ccf97dccc2f1fa4cf045134e227859e4a55d524 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 6 Jun 2026 23:57:30 +0100 Subject: [PATCH 22/30] refactor: move shared format/schema types to types.rs (decouple from DuckDB engine) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces src/types.rs with InputFormat, ExportFormat, CompressionCodec, and SchemaColumn — four engine-agnostic leaf types previously defined in engine.rs. All importers (engine.rs, polars_engine.rs, output_writer.rs, profile_command.rs, query_pipeline.rs, reference_tables.rs, schema.rs) now resolve these from crate::types. DuckDB engine unchanged; all 146 tests pass, clippy clean. Co-Authored-By: Claude Sonnet 4.6 --- src/engine.rs | 32 +------------------------------- src/main.rs | 2 ++ src/output_writer.rs | 3 ++- src/polars_engine.rs | 2 +- src/profile_command.rs | 3 ++- src/query_pipeline.rs | 5 ++--- src/reference_tables.rs | 3 ++- src/schema.rs | 7 +++---- src/types.rs | 32 ++++++++++++++++++++++++++++++++ 9 files changed, 47 insertions(+), 42 deletions(-) create mode 100644 src/types.rs diff --git a/src/engine.rs b/src/engine.rs index 99b2cd3..cab0576 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -5,32 +5,9 @@ use duckdb::{Connection, arrow::record_batch::RecordBatch, types::Value}; use crate::{ error::DtooError, sql_utils::{escape_sql_literal, quote_identifier}, + types::{CompressionCodec, ExportFormat, InputFormat, SchemaColumn}, }; -/// Input file format for registering the magic `_` view. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum InputFormat { - Parquet, - Csv { delimiter: char }, - Ndjson, - Excel { sheet: Option }, -} - -/// Export format for writing `temp_results`. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum ExportFormat { - Csv, - Parquet, - Ndjson, -} - -/// Compression codec for export operations. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum CompressionCodec { - Gzip, - Zstd, -} - /// Optional cloud settings applied during engine initialisation. #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct CloudSettings { @@ -68,13 +45,6 @@ impl ArrowBatch { } } -/// One explicit schema column used to build `temp_results`. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct SchemaColumn { - pub name: String, - pub data_type: String, -} - /// Thin wrapper around an in-memory DuckDB connection. pub struct DuckDbEngine { connection: Connection, diff --git a/src/main.rs b/src/main.rs index 3ac949a..9919d4b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,6 +39,8 @@ mod reference_tables; mod schema; #[allow(dead_code)] mod sql_utils; +#[allow(dead_code)] +mod types; use std::process::ExitCode; diff --git a/src/output_writer.rs b/src/output_writer.rs index 3391413..76e98b5 100644 --- a/src/output_writer.rs +++ b/src/output_writer.rs @@ -1,8 +1,9 @@ use std::path::{Path, PathBuf}; use crate::{ - engine::{CompressionCodec, DuckDbEngine, ExportFormat}, + engine::DuckDbEngine, error::DtooError, + types::{CompressionCodec, ExportFormat}, }; /// Output writer configuration for exporting `temp_results`. diff --git a/src/polars_engine.rs b/src/polars_engine.rs index dfb2bb1..78a6845 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -8,8 +8,8 @@ use std::path::Path; use polars::prelude::*; -use crate::engine::{CompressionCodec, ExportFormat, InputFormat}; use crate::error::DtooError; +use crate::types::{CompressionCodec, ExportFormat, InputFormat}; /// Stateless handle for Polars-backed data operations. pub struct PolarsEngine; diff --git a/src/profile_command.rs b/src/profile_command.rs index 6bea8dd..6c54d7d 100644 --- a/src/profile_command.rs +++ b/src/profile_command.rs @@ -3,11 +3,12 @@ use std::path::PathBuf; use crate::{ cli::{OnErrorMode, PipeMode, ProfileArgs, StdinFormat}, crypto, - engine::{CloudSettings, DuckDbEngine, EngineConfig, InputFormat}, + engine::{CloudSettings, DuckDbEngine, EngineConfig}, error::DtooError, file_resolution::{FileFormat, FileResolver, FileResolverConfig}, path_utils::is_cloud_path, profiler::{ProfileOptions, Profiler}, + types::InputFormat, }; pub fn run(args: &ProfileArgs) -> Result<(), DtooError> { diff --git a/src/query_pipeline.rs b/src/query_pipeline.rs index c2b4b0f..14156dd 100644 --- a/src/query_pipeline.rs +++ b/src/query_pipeline.rs @@ -5,9 +5,7 @@ use uuid::Uuid; use crate::{ cli::{CompressMethod, OnErrorMode, OutputFormat, PipeMode, QueryArgs, StdinFormat}, crypto, - engine::{ - CloudSettings, CompressionCodec, DuckDbEngine, EngineConfig, ExportFormat, InputFormat, - }, + engine::{CloudSettings, DuckDbEngine, EngineConfig}, error::DtooError, file_resolution::{FileFormat, FileResolver, FileResolverConfig, ResolutionReport}, fingerprint::fingerprint_file, @@ -22,6 +20,7 @@ use crate::{ profiler::{ProfileOptions, Profiler}, reference_tables::{load_reference_tables, parse_reference_tables}, schema::SchemaManager, + types::{CompressionCodec, ExportFormat, InputFormat}, }; /// Coordinates execution of the `dtoo query` pipeline. diff --git a/src/reference_tables.rs b/src/reference_tables.rs index bcc170e..2bdcbda 100644 --- a/src/reference_tables.rs +++ b/src/reference_tables.rs @@ -1,9 +1,10 @@ use std::{collections::HashSet, path::Path}; use crate::{ - engine::{DuckDbEngine, InputFormat}, + engine::DuckDbEngine, error::DtooError, path_utils::{is_cloud_path, split_excel_sheet_from_path}, + types::InputFormat, }; /// One parsed `--ref NAME=PATH` entry. diff --git a/src/schema.rs b/src/schema.rs index 5aeebac..fce05c0 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -7,9 +7,7 @@ use std::{ use serde::Deserialize; use crate::{ - engine::{DuckDbEngine, SchemaColumn}, - error::DtooError, - sql_utils::quote_identifier, + engine::DuckDbEngine, error::DtooError, sql_utils::quote_identifier, types::SchemaColumn, }; /// Handles auto-detected or explicit schema setup for `temp_results`. @@ -213,7 +211,8 @@ fn is_valid_duckdb_identifier(name: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::engine::{CloudSettings, EngineConfig, InputFormat}; + use crate::engine::{CloudSettings, EngineConfig}; + use crate::types::InputFormat; use std::sync::atomic::{AtomicU64, Ordering}; static UNIQUE_COUNTER: AtomicU64 = AtomicU64::new(0); diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000..e3ed4d3 --- /dev/null +++ b/src/types.rs @@ -0,0 +1,32 @@ +//! Engine-agnostic data types shared across the dtoo pipeline. + +/// Input file format for registering the magic `_` view. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum InputFormat { + Parquet, + Csv { delimiter: char }, + Ndjson, + Excel { sheet: Option }, +} + +/// Export format for writing `temp_results`. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum ExportFormat { + Csv, + Parquet, + Ndjson, +} + +/// Compression codec for export operations. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum CompressionCodec { + Gzip, + Zstd, +} + +/// One explicit schema column used to build `temp_results`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SchemaColumn { + pub name: String, + pub data_type: String, +} From 8ae8c76ffda96c5ee995303b1e91b6c4ed06f54e Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sun, 7 Jun 2026 00:00:52 +0100 Subject: [PATCH 23/30] feat: masking as native Polars DataFrame transform Add `mask_dataframe` to apply deterministic sha256 column masking directly on a Polars DataFrame without going through DuckDB SQL. Existing `MaskingEngine` retained for the DuckDB pipeline until teardown. Co-Authored-By: Claude Sonnet 4.6 --- src/masking.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/masking.rs b/src/masking.rs index c2f1708..e1ddedb 100644 --- a/src/masking.rs +++ b/src/masking.rs @@ -1,11 +1,73 @@ use std::collections::HashSet; +use polars::prelude::*; +use sha2::{Digest, Sha256}; + use crate::{ engine::DuckDbEngine, error::DtooError, sql_utils::{escape_sql_literal, quote_identifier}, }; +/// Replace each selected column's non-null values with `hex(sha256("{salt}:{col}:" + value))`. +/// +/// NULLs are preserved. Unknown column names produce a [`DtooError::Config`]. +pub fn mask_dataframe( + mut df: DataFrame, + columns: &[String], + salt: &str, +) -> Result { + if columns.is_empty() { + return Ok(df); + } + let available: HashSet = df + .get_column_names() + .iter() + .map(|s| s.to_string()) + .collect(); + for column in columns { + if !available.contains(column) { + let mut sorted: Vec = available.iter().cloned().collect(); + sorted.sort(); + return Err(DtooError::Config { + message: format!( + "mask column `{column}` not found. available columns: {}", + sorted.join(", ") + ), + }); + } + let prefix = format!("{salt}:{column}:"); + let as_str = df + .column(column) + .map_err(|e| DtooError::Config { + message: e.to_string(), + })? + .cast(&DataType::String) + .map_err(|e| DtooError::Config { + message: e.to_string(), + })?; + let chunked = as_str.str().map_err(|e| DtooError::Config { + message: e.to_string(), + })?; + let masked: StringChunked = chunked + .iter() + .map(|opt| { + opt.map(|v| { + let mut h = Sha256::new(); + h.update(prefix.as_bytes()); + h.update(v.as_bytes()); + hex::encode(h.finalize()) + }) + }) + .collect(); + df.replace(column, masked.with_name(column.into()).into_column()) + .map_err(|e| DtooError::Config { + message: e.to_string(), + })?; + } + Ok(df) +} + /// Applies deterministic masking updates to selected columns. #[derive(Clone, Debug)] pub struct MaskingEngine { @@ -132,4 +194,27 @@ mod tests { let err = masking.apply(&engine).expect_err("should fail"); assert!(matches!(err, DtooError::Config { .. })); } + + #[test] + fn mask_columns_is_deterministic_and_preserves_null() { + use polars::prelude::*; + let df = df![ + "email" => [Some("a@example.com"), Some("a@example.com"), None] + ] + .unwrap(); + let out = mask_dataframe(df, &["email".to_string()], "project-x").unwrap(); + let binding = out.column("email").unwrap(); + let col = binding.str().unwrap(); + assert_eq!(col.get(0), col.get(1)); // deterministic + assert!(col.get(2).is_none()); // null preserved + assert_ne!(col.get(0), Some("a@example.com")); // actually hashed + } + + #[test] + fn mask_dataframe_unknown_column_errors() { + use polars::prelude::*; + let df = df!["email" => ["x"]].unwrap(); + let err = mask_dataframe(df, &["missing".to_string()], "").unwrap_err(); + assert!(matches!(err, DtooError::Config { .. })); + } } From 0b04b43ac97a1e3eb599257f378d2c9c9e7089e6 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sun, 7 Jun 2026 00:03:20 +0100 Subject: [PATCH 24/30] feat: lineage columns as native Polars DataFrame transforms Add `apply_to_dataframe` to `LineageManager` that applies all requested lineage columns directly to a Polars DataFrame without going through DuckDB SQL. Existing `apply_columns` and `tag_rows_with_origin` are preserved for the DuckDB pipeline until teardown. Co-Authored-By: Claude Sonnet 4.6 --- src/lineage.rs | 102 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/lineage.rs b/src/lineage.rs index 7b05646..5743268 100644 --- a/src/lineage.rs +++ b/src/lineage.rs @@ -155,6 +155,73 @@ impl LineageManager { pub fn batch_timestamp(&self) -> DateTime { self.batch_timestamp } + + /// Apply requested lineage columns to a DataFrame (native Polars). + pub fn apply_to_dataframe( + &self, + mut df: polars::prelude::DataFrame, + ) -> Result { + use polars::prelude::*; + + if self.requested.is_empty() { + return Ok(df); + } + + let n = df.height(); + let schema_err = |e: PolarsError| DtooError::Schema { + message: e.to_string(), + }; + + if self.requested.contains(&LineageColumn::BatchId) { + df.with_column(Series::new("batch_id".into(), vec![self.batch_id.clone(); n]).into()) + .map_err(schema_err)?; + } + + if self.requested.contains(&LineageColumn::RecordId) { + let ids: Vec = (0..n).map(|_| Uuid::new_v4().to_string()).collect(); + df.with_column(Series::new("record_id".into(), ids).into()) + .map_err(schema_err)?; + } + + if self.requested.contains(&LineageColumn::BatchTimestamp) { + df.with_column( + Series::new( + "batch_timestamp".into(), + vec![self.batch_timestamp.to_rfc3339(); n], + ) + .into(), + ) + .map_err(schema_err)?; + } + + if self.requested.contains(&LineageColumn::BatchHash) { + df.with_column( + Series::new("batch_hash".into(), vec![self.batch_hash.clone(); n]).into(), + ) + .map_err(schema_err)?; + } + + let has_origin = df + .get_column_names() + .iter() + .any(|c| c.as_str() == "_origin_file"); + + if self.requested.contains(&LineageColumn::OriginFile) { + if !has_origin { + return Err(DtooError::Schema { + message: + "origin_file lineage requested but internal _origin_file column is missing" + .to_string(), + }); + } + df.rename("_origin_file", "origin_file".into()) + .map_err(schema_err)?; + } else if has_origin { + df = df.drop("_origin_file").map_err(schema_err)?; + } + + Ok(df) + } } fn parse_requested_columns(lineage: Option<&str>) -> Result, DtooError> { @@ -217,6 +284,41 @@ mod tests { use super::*; use crate::engine::{CloudSettings, EngineConfig}; + #[test] + fn apply_lineage_adds_requested_columns_and_renames_origin() { + use polars::prelude::*; + let df = df!["id" => [1i64], "_origin_file" => ["/tmp/a.csv"]].unwrap(); + let mgr = LineageManager::new( + Some("batch_id,record_id,origin_file"), + LineageContext { + files: vec!["/tmp/a.csv".to_string()], + ..LineageContext::default() + }, + ) + .unwrap(); + let out = mgr.apply_to_dataframe(df).unwrap(); + let names = out.get_column_names(); + assert!(names.iter().any(|n| n.as_str() == "batch_id")); + assert!(names.iter().any(|n| n.as_str() == "record_id")); + assert!(names.iter().any(|n| n.as_str() == "origin_file")); + assert!(!names.iter().any(|n| n.as_str() == "_origin_file")); + assert_eq!( + out.column("origin_file").unwrap().str().unwrap().get(0), + Some("/tmp/a.csv") + ); + } + + #[test] + fn apply_lineage_origin_requested_but_missing_errors() { + use polars::prelude::*; + let df = df!["id" => [1i64]].unwrap(); + let mgr = LineageManager::new(Some("origin_file"), LineageContext::default()).unwrap(); + assert!(matches!( + mgr.apply_to_dataframe(df), + Err(DtooError::Schema { .. }) + )); + } + #[test] fn batch_hash_is_deterministic_for_same_context() { let context = LineageContext { From 62e476a5f73d8d6c9f9191d8ac08965226a2b33d Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sun, 7 Jun 2026 00:06:39 +0100 Subject: [PATCH 25/30] feat: explicit-schema coercion as Polars cast/project with type map Add duckdb_type_to_polars() mapping DuckDB type strings to Polars DataType, and coerce_to_schema() projecting a LazyFrame to declared columns with casts, NULL fill for missing columns, and dropping undeclared source columns. Co-Authored-By: Claude Sonnet 4.6 --- src/schema.rs | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/src/schema.rs b/src/schema.rs index fce05c0..8111b69 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -4,6 +4,7 @@ use std::{ path::Path, }; +use polars::prelude::{DataType, Expr, LazyFrame, LiteralValue, PlSmallStr, TimeUnit, col, lit}; use serde::Deserialize; use crate::{ @@ -97,6 +98,82 @@ impl SchemaManager { } } +/// Map a DuckDB-style type string to a Polars [`DataType`]. +/// +/// Case-insensitive; tolerates `DECIMAL(p,s)` and `NUMERIC(p,s)` parameterised forms. +/// Returns [`DtooError::Schema`] for unknown or unsupported type names. +pub fn duckdb_type_to_polars(data_type: &str) -> Result { + let upper = data_type.trim().to_ascii_uppercase(); + let base = upper.split('(').next().unwrap_or("").trim(); + let dt = match base { + "INTEGER" | "INT" | "INT4" => DataType::Int32, + "BIGINT" | "INT8" | "LONG" => DataType::Int64, + "SMALLINT" | "INT2" => DataType::Int16, + "TINYINT" => DataType::Int8, + "DOUBLE" | "FLOAT8" => DataType::Float64, + "REAL" | "FLOAT" | "FLOAT4" => DataType::Float32, + "BOOLEAN" | "BOOL" => DataType::Boolean, + "VARCHAR" | "TEXT" | "STRING" | "CHAR" => DataType::String, + "DATE" => DataType::Date, + "TIMESTAMP" | "DATETIME" => DataType::Datetime(TimeUnit::Microseconds, None), + "DECIMAL" | "NUMERIC" => { + let (p, s) = parse_decimal_params(&upper); + // Polars 0.54 Decimal(usize, usize): default precision=38, scale=0 + DataType::Decimal(p.unwrap_or(38), s.unwrap_or(0)) + } + _ => { + return Err(DtooError::Schema { + message: format!("unsupported schema type `{data_type}`"), + }); + } + }; + Ok(dt) +} + +fn parse_decimal_params(upper: &str) -> (Option, Option) { + if let Some(open) = upper.find('(') + && let Some(close) = upper[open..].find(')') + { + let inner = &upper[open + 1..open + close]; + let mut parts = inner.split(',').map(|x| x.trim().parse::().ok()); + let p = parts.next().flatten(); + let s = parts.next().flatten(); + return (p, s); + } + (None, None) +} + +/// Project a [`LazyFrame`] to the declared columns, casting present columns to +/// declared Polars types and filling absent columns with typed `NULL`s. +/// +/// - Output columns appear in exactly the declared order. +/// - Source columns not present in `columns` are dropped. +/// - A declared column absent from the source produces a `NULL` column of the declared type. +pub fn coerce_to_schema(lf: LazyFrame, columns: &[SchemaColumn]) -> Result { + let schema = lf.clone().collect_schema().map_err(|e| DtooError::Schema { + message: e.to_string(), + })?; + + let present: std::collections::HashSet = schema + .iter_names() + .map(|n: &PlSmallStr| n.to_string()) + .collect(); + + let mut exprs: Vec = Vec::with_capacity(columns.len()); + for c in columns { + let dt = duckdb_type_to_polars(&c.data_type)?; + let e = if present.contains(&c.name) { + col(c.name.as_str()).cast(dt).alias(c.name.as_str()) + } else { + lit(LiteralValue::untyped_null()) + .cast(dt) + .alias(c.name.as_str()) + }; + exprs.push(e); + } + Ok(lf.select(exprs)) +} + fn projected_query_for_schema( engine: &DuckDbEngine, source_query: &str, @@ -358,6 +435,43 @@ mod tests { let _ = fs::remove_file(csv); } + #[test] + fn duckdb_type_maps_to_polars() { + use polars::prelude::*; + assert_eq!(duckdb_type_to_polars("INTEGER").unwrap(), DataType::Int32); + assert_eq!(duckdb_type_to_polars("BIGINT").unwrap(), DataType::Int64); + assert_eq!(duckdb_type_to_polars("VARCHAR").unwrap(), DataType::String); + assert_eq!(duckdb_type_to_polars("BOOLEAN").unwrap(), DataType::Boolean); + assert!(duckdb_type_to_polars("NOPE_TYPE").is_err()); + } + + #[test] + fn coerce_projects_casts_and_nulls_missing() { + use crate::types::SchemaColumn; + use polars::prelude::*; + let lf = df!["id" => ["1"], "extra" => ["x"]].unwrap().lazy(); + let cols = vec![ + SchemaColumn { + name: "id".into(), + data_type: "INTEGER".into(), + }, + SchemaColumn { + name: "name".into(), + data_type: "VARCHAR".into(), + }, + ]; + let out = coerce_to_schema(lf, &cols).unwrap().collect().unwrap(); + assert_eq!( + out.get_column_names() + .iter() + .map(|s| s.as_str()) + .collect::>(), + vec!["id", "name"] + ); // declared order, extra dropped + assert_eq!(out.column("id").unwrap().dtype(), &DataType::Int32); + assert_eq!(out.column("name").unwrap().null_count(), 1); // missing -> null + } + fn temp_schema(contents: &str) -> std::path::PathBuf { let path = std::env::temp_dir().join(format!("dtoo-schema-{}.yaml", unique_suffix())); fs::write(&path, contents).expect("write schema file"); From 490f303ebcff5237dfa0f669aa6903dfea44d39a Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sun, 7 Jun 2026 00:08:42 +0100 Subject: [PATCH 26/30] feat: reference tables loaded as Polars LazyFrames Add `load_reference_lazyframes` to `src/reference_tables.rs`, which scans each `ReferenceTable` into a `(name, LazyFrame)` pair via `PolarsEngine::scan`. The existing DuckDB `load_reference_tables` is untouched (still used by the DuckDB pipeline until teardown). Co-Authored-By: Claude Sonnet 4.6 --- src/reference_tables.rs | 44 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/reference_tables.rs b/src/reference_tables.rs index 2bdcbda..01f73a0 100644 --- a/src/reference_tables.rs +++ b/src/reference_tables.rs @@ -1,9 +1,12 @@ use std::{collections::HashSet, path::Path}; +use polars::prelude::LazyFrame; + use crate::{ engine::DuckDbEngine, error::DtooError, path_utils::{is_cloud_path, split_excel_sheet_from_path}, + polars_engine::PolarsEngine, types::InputFormat, }; @@ -94,6 +97,19 @@ pub fn load_reference_tables( Ok(loaded) } +/// Load each reference table as a `(name, LazyFrame)` pair via the Polars engine. +pub fn load_reference_lazyframes( + engine: &PolarsEngine, + refs: &[ReferenceTable], +) -> Result, DtooError> { + let mut loaded = Vec::with_capacity(refs.len()); + for spec in refs { + let lf = engine.scan(&spec.path, &spec.format)?; + loaded.push((spec.name.clone(), lf)); + } + Ok(loaded) +} + fn is_valid_identifier(name: &str) -> bool { let mut chars = name.chars(); let Some(first) = chars.next() else { @@ -247,4 +263,32 @@ mod tests { let counter = UNIQUE_COUNTER.fetch_add(1, Ordering::Relaxed); std::env::temp_dir().join(format!("dtoo-{prefix}-{nanos}-{counter}.{ext}")) } + + #[test] + fn load_reference_lazyframes_loads_csv_with_row_count() { + use crate::polars_engine::PolarsEngine; + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "dtoo-ref-{}.csv", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&path, "id,region_name\n10,EMEA\n20,APAC\n").unwrap(); + + let refs = vec![ReferenceTable { + name: "regions".to_string(), + path: path.to_string_lossy().to_string(), + format: crate::types::InputFormat::Csv { delimiter: ',' }, + }]; + let engine = PolarsEngine::new(); + let loaded = load_reference_lazyframes(&engine, &refs).unwrap(); + + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].0, "regions"); + // second tuple element is a LazyFrame; collecting it yields 2 rows + assert_eq!(engine.collect(loaded[0].1.clone()).unwrap().height(), 2); + let _ = std::fs::remove_file(path); + } } From 722d8f21132bf6db83570fe0d71870054ee8e6f5 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sun, 7 Jun 2026 00:52:09 +0100 Subject: [PATCH 27/30] feat: profiler computes statistics from a Polars DataFrame Add `Profiler::generate(df, options)` that profiles a Polars DataFrame directly: per-column null%, distinct count, top-5 values, numeric quantiles/mean/stddev/median, string char-lengths and patterns, and date min/max. The original DuckDB-based path is preserved as `generate_from_engine` so query_pipeline and profile_command continue to compile until Task 9/12 completes the pipeline cutover. Co-Authored-By: Claude Sonnet 4.6 --- src/profile_command.rs | 2 +- src/profiler.rs | 471 ++++++++++++++++++++++++++++++++++------- src/query_pipeline.rs | 2 +- 3 files changed, 401 insertions(+), 74 deletions(-) diff --git a/src/profile_command.rs b/src/profile_command.rs index 6c54d7d..1705955 100644 --- a/src/profile_command.rs +++ b/src/profile_command.rs @@ -62,7 +62,7 @@ pub fn run(args: &ProfileArgs) -> Result<(), DtooError> { format: args.format, sample_percentage: args.sample, }; - Profiler::generate(&engine, &options) + Profiler::generate_from_engine(&engine, &options) } fn to_input_format(format: &FileFormat, sheet: Option<&str>) -> InputFormat { diff --git a/src/profiler.rs b/src/profiler.rs index 967bf8e..e9561cb 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -1,9 +1,11 @@ use std::{ + collections::HashMap, fs, path::{Path, PathBuf}, }; use chrono::Utc; +use polars::prelude::*; use serde::Serialize; use crate::{ @@ -54,11 +56,44 @@ pub struct ProfileOptions { pub sample_percentage: u8, } -/// Computes and renders profile reports from `temp_results`. +/// Computes and renders profile reports from a [`DataFrame`] or from `temp_results` (legacy). pub struct Profiler; impl Profiler { - pub fn generate(engine: &DuckDbEngine, options: &ProfileOptions) -> Result<(), DtooError> { + /// Compute a profile report from a Polars [`DataFrame`] and write it according to `options`. + /// + /// Sampling uses `df.head(n)` (deterministic, take the first N rows) when + /// `sample_percentage < 100`. + pub fn generate(df: &DataFrame, options: &ProfileOptions) -> Result<(), DtooError> { + if options.sample_percentage == 0 || options.sample_percentage > 100 { + return Err(DtooError::Config { + message: "--profile-sample must be between 1 and 100".to_string(), + }); + } + + let sampled: DataFrame; + let source: &DataFrame = if options.sample_percentage < 100 { + let n = ((df.height() as f64 * options.sample_percentage as f64 / 100.0).round() + as usize) + .max(1); + sampled = df.head(Some(n)); + &sampled + } else { + df + }; + + let report = build_report(source, options.sample_percentage)?; + write_report(options, &report) + } + + /// Legacy path: compute a profile report from the DuckDB `temp_results` table. + /// + /// Retained for Task-9 compatibility; will be removed when the pipeline is + /// fully ported to Polars (Phase 2, Task 12). + pub fn generate_from_engine( + engine: &DuckDbEngine, + options: &ProfileOptions, + ) -> Result<(), DtooError> { if options.sample_percentage == 0 || options.sample_percentage > 100 { return Err(DtooError::Config { message: "--profile-sample must be between 1 and 100".to_string(), @@ -78,7 +113,7 @@ impl Profiler { "temp_results" }; - let report_result = build_report(engine, source, options.sample_percentage); + let report_result = build_report_from_engine(engine, source, options.sample_percentage); if sampled { let _ = engine.execute("DROP VIEW IF EXISTS _profile_source"); } @@ -87,7 +122,247 @@ impl Profiler { } } -fn build_report( +// ── Polars-based report builder ─────────────────────────────────────────────── + +fn build_report(df: &DataFrame, sample_percentage: u8) -> Result { + let row_count = df.height(); + let mut columns = Vec::with_capacity(df.width()); + + for col in df.columns() { + columns.push(profile_column(col, row_count)?); + } + + Ok(ProfileReport { + row_count, + sample_percentage, + generated_at: Utc::now().to_rfc3339(), + columns, + }) +} + +fn profile_column(series: &Column, total_rows: usize) -> Result { + let name = series.name().to_string(); + let dtype = series.dtype().clone(); + let data_type = format!("{dtype:?}"); + + let null_count = series.null_count(); + let count = total_rows; + let null_percentage = if count == 0 { + 0.0 + } else { + (100.0 * null_count as f64 / count as f64 * 100.0).round() / 100.0 + }; + let distinct_count = series.n_unique().map_err(polars_err)?; + + let top_5 = top_values(series)?; + + let mut profile = ColumnProfile { + name, + data_type, + count, + null_count, + null_percentage, + distinct_count, + min: None, + max: None, + mean: None, + stddev: None, + median: None, + p25: None, + p75: None, + min_length: None, + max_length: None, + avg_length: None, + top_5_values: top_5, + pattern_sample: Vec::new(), + }; + + if is_numeric_dtype(&dtype) { + profile.min = scalar_to_opt_string(series.min_reduce().map_err(polars_err)?); + profile.max = scalar_to_opt_string(series.max_reduce().map_err(polars_err)?); + profile.mean = scalar_to_opt_string(series.mean_reduce().map_err(polars_err)?); + profile.stddev = scalar_to_opt_string(series.std_reduce(1).map_err(polars_err)?); + profile.median = scalar_to_opt_string(series.median_reduce().map_err(polars_err)?); + profile.p25 = scalar_to_opt_string( + series + .quantile_reduce(0.25, QuantileMethod::Linear) + .map_err(polars_err)?, + ); + profile.p75 = scalar_to_opt_string( + series + .quantile_reduce(0.75, QuantileMethod::Linear) + .map_err(polars_err)?, + ); + } else if is_text_dtype(&dtype) { + let lengths = string_char_lengths(series)?; + if !lengths.is_empty() { + let min_l = lengths.iter().copied().min().unwrap_or(0); + let max_l = lengths.iter().copied().max().unwrap_or(0); + let avg_l = lengths.iter().copied().sum::() as f64 / lengths.len() as f64; + profile.min_length = Some(min_l.to_string()); + profile.max_length = Some(max_l.to_string()); + profile.avg_length = Some(format!("{avg_l:.2}")); + } + profile.pattern_sample = text_patterns(series)?; + } else if is_date_like_dtype(&dtype) { + // For date/time types, cast to String for human-readable output before reducing. + let as_str = series.cast(&DataType::String).map_err(polars_err)?; + profile.min = scalar_to_opt_string(as_str.min_reduce().map_err(polars_err)?); + profile.max = scalar_to_opt_string(as_str.max_reduce().map_err(polars_err)?); + } + + Ok(profile) +} + +/// Returns the character lengths of all non-null string values. +fn string_char_lengths(series: &Column) -> Result, DtooError> { + let as_str = series.cast(&DataType::String).map_err(polars_err)?; + let ca = as_str.str().map_err(polars_err)?; + let lengths: Vec = ca + .str_len_chars() + .iter() + .flatten() // drops None (nulls) + .collect(); + Ok(lengths) +} + +/// Returns the top-5 most frequent non-null values, sorted descending by count. +fn top_values(series: &Column) -> Result, DtooError> { + // Cast to String so all types produce a uniform representation. + let as_str = series.cast(&DataType::String).map_err(polars_err)?; + let non_null = as_str.drop_nulls(); + + // value_counts(sort=true, parallel=false, name="count", normalize=false) + let vc_df = non_null + .as_materialized_series() + .value_counts(true, false, "count".into(), false) + .map_err(polars_err)?; + + // Columns: [series_name (String), "count" (UInt32)] + let values_col = vc_df.column(series.name()).map_err(polars_err)?; + let counts_col = vc_df.column("count").map_err(polars_err)?; + + let values_ca = values_col.str().map_err(polars_err)?; + let counts_ca = counts_col.cast(&DataType::UInt64).map_err(polars_err)?; + let counts_u64 = counts_ca.u64().map_err(polars_err)?; + + let mut pairs: Vec = values_ca + .iter() + .zip(counts_u64.iter()) + .filter_map(|(v, c)| { + Some(ValueFrequency { + value: v?.to_string(), + freq: c? as usize, + }) + }) + .collect(); + + // value_counts with sort=true returns descending; take at most 5. + pairs.truncate(5); + Ok(pairs) +} + +/// Computes text patterns: replace digits→`d`, letters→`a`, collapse `d+`→`N`; +/// returns the top-5 patterns by frequency. +fn text_patterns(series: &Column) -> Result, DtooError> { + let as_str = series.cast(&DataType::String).map_err(polars_err)?; + let ca = as_str.str().map_err(polars_err)?; + + let mut freq: HashMap = HashMap::new(); + for v in ca.iter().flatten() { + let pattern = make_pattern(v); + *freq.entry(pattern).or_insert(0) += 1; + } + + let mut pairs: Vec = freq + .into_iter() + .map(|(value, freq)| ValueFrequency { value, freq }) + .collect(); + pairs.sort_by_key(|b| std::cmp::Reverse(b.freq)); + pairs.truncate(5); + Ok(pairs) +} + +/// Replace each digit with `d`, each ASCII letter with `a`, then collapse +/// runs of consecutive `d` characters into the single token `N`. +fn make_pattern(s: &str) -> String { + // Step 1: replace each digit → 'd', each ASCII letter → 'a' + let replaced: String = s + .chars() + .map(|c| { + if c.is_ascii_digit() { + 'd' + } else if c.is_ascii_alphabetic() { + 'a' + } else { + c + } + }) + .collect(); + + // Step 2: collapse runs of 'd' → 'N' + let mut result = String::with_capacity(replaced.len()); + let mut in_digit_run = false; + for c in replaced.chars() { + if c == 'd' { + if !in_digit_run { + result.push('N'); + in_digit_run = true; + } + } else { + in_digit_run = false; + result.push(c); + } + } + result +} + +fn scalar_to_opt_string(scalar: Scalar) -> Option { + let av = scalar.value(); + if matches!(av, AnyValue::Null) { + None + } else { + Some(format!("{av}")) + } +} + +fn polars_err(e: PolarsError) -> DtooError { + DtooError::Config { + message: format!("profiler: {e}"), + } +} + +fn is_numeric_dtype(dt: &DataType) -> bool { + matches!( + dt, + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float32 + | DataType::Float64 + | DataType::Decimal(_, _) + ) +} + +fn is_text_dtype(dt: &DataType) -> bool { + matches!(dt, DataType::String) +} + +fn is_date_like_dtype(dt: &DataType) -> bool { + matches!( + dt, + DataType::Date | DataType::Datetime(_, _) | DataType::Time | DataType::Duration(_) + ) +} + +// ── DuckDB-based report builder (legacy, kept for Task-9 bridge) ───────────── + +fn build_report_from_engine( engine: &DuckDbEngine, source_table: &str, sample_percentage: u8, @@ -104,7 +379,12 @@ fn build_report( continue; }; let data_type = row.values.get(1).cloned().unwrap_or_default(); - columns.push(profile_column(engine, source_table, name, &data_type)?); + columns.push(profile_column_from_engine( + engine, + source_table, + name, + &data_type, + )?); } Ok(ProfileReport { @@ -115,7 +395,7 @@ fn build_report( }) } -fn profile_column( +fn profile_column_from_engine( engine: &DuckDbEngine, source_table: &str, column_name: &str, @@ -156,41 +436,41 @@ fn profile_column( min_length: None, max_length: None, avg_length: None, - top_5_values: top_values(engine, source_table, &col)?, + top_5_values: top_values_from_engine(engine, source_table, &col)?, pattern_sample: Vec::new(), }; - if is_numeric(data_type) { + if is_numeric_str(data_type) { let rows = engine.query(&format!( "SELECT MIN({col}), MAX({col}), AVG({col}), STDDEV({col}), MEDIAN({col}), QUANTILE_CONT({col}, 0.25), QUANTILE_CONT({col}, 0.75) FROM {source_table}" ))?; - profile.min = value_at(&rows, 0); - profile.max = value_at(&rows, 1); - profile.mean = value_at(&rows, 2); - profile.stddev = value_at(&rows, 3); - profile.median = value_at(&rows, 4); - profile.p25 = value_at(&rows, 5); - profile.p75 = value_at(&rows, 6); - } else if is_text(data_type) { + profile.min = value_at_engine(&rows, 0); + profile.max = value_at_engine(&rows, 1); + profile.mean = value_at_engine(&rows, 2); + profile.stddev = value_at_engine(&rows, 3); + profile.median = value_at_engine(&rows, 4); + profile.p25 = value_at_engine(&rows, 5); + profile.p75 = value_at_engine(&rows, 6); + } else if is_text_str(data_type) { let rows = engine.query(&format!( "SELECT MIN(LENGTH({col})), MAX(LENGTH({col})), AVG(LENGTH({col})) FROM {source_table} WHERE {col} IS NOT NULL" ))?; - profile.min_length = value_at(&rows, 0); - profile.max_length = value_at(&rows, 1); - profile.avg_length = value_at(&rows, 2); - profile.pattern_sample = text_patterns(engine, source_table, &col)?; - } else if is_date_like(data_type) { + profile.min_length = value_at_engine(&rows, 0); + profile.max_length = value_at_engine(&rows, 1); + profile.avg_length = value_at_engine(&rows, 2); + profile.pattern_sample = text_patterns_from_engine(engine, source_table, &col)?; + } else if is_date_like_str(data_type) { let rows = engine.query(&format!( "SELECT MIN({col}), MAX({col}) FROM {source_table}" ))?; - profile.min = value_at(&rows, 0); - profile.max = value_at(&rows, 1); + profile.min = value_at_engine(&rows, 0); + profile.max = value_at_engine(&rows, 1); } Ok(profile) } -fn top_values( +fn top_values_from_engine( engine: &DuckDbEngine, source_table: &str, col: &str, @@ -211,7 +491,7 @@ fn top_values( .collect()) } -fn text_patterns( +fn text_patterns_from_engine( engine: &DuckDbEngine, source_table: &str, col: &str, @@ -232,13 +512,35 @@ fn text_patterns( .collect()) } -fn value_at(rows: &[crate::engine::QueryRow], idx: usize) -> Option { +fn value_at_engine(rows: &[crate::engine::QueryRow], idx: usize) -> Option { rows.first() .and_then(|r| r.values.get(idx)) .filter(|v| *v != "NULL") .cloned() } +fn is_numeric_str(data_type: &str) -> bool { + let u = data_type.to_ascii_uppercase(); + ["INT", "DOUBLE", "FLOAT", "DECIMAL", "NUMERIC"] + .iter() + .any(|token| u.contains(token)) + && !u.contains("INTERVAL") +} + +fn is_text_str(data_type: &str) -> bool { + let u = data_type.to_ascii_uppercase(); + ["CHAR", "TEXT", "VARCHAR", "STRING"] + .iter() + .any(|token| u.contains(token)) +} + +fn is_date_like_str(data_type: &str) -> bool { + let u = data_type.to_ascii_uppercase(); + u.contains("DATE") || u.contains("TIME") +} + +// ── Shared rendering (UNCHANGED) ───────────────────────────────────────────── + fn write_report(options: &ProfileOptions, report: &ProfileReport) -> Result<(), DtooError> { let content = match options.format { ProfileFormat::Json => { @@ -425,50 +727,17 @@ fn csv_escape(value: &str) -> String { } } -fn is_numeric(data_type: &str) -> bool { - let u = data_type.to_ascii_uppercase(); - ["INT", "DOUBLE", "FLOAT", "DECIMAL", "NUMERIC"] - .iter() - .any(|token| u.contains(token)) - && !u.contains("INTERVAL") -} - -fn is_text(data_type: &str) -> bool { - let u = data_type.to_ascii_uppercase(); - ["CHAR", "TEXT", "VARCHAR", "STRING"] - .iter() - .any(|token| u.contains(token)) -} - -fn is_date_like(data_type: &str) -> bool { - let u = data_type.to_ascii_uppercase(); - u.contains("DATE") || u.contains("TIME") -} - #[cfg(test)] mod tests { use super::*; - use crate::engine::{CloudSettings, EngineConfig}; - - fn test_engine() -> DuckDbEngine { - DuckDbEngine::new(EngineConfig { - cloud: CloudSettings::default(), - load_cloud_extensions: false, - load_excel_extension: false, - }) - .expect("engine init") - } #[test] fn generates_json_profile_file() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (id INTEGER, email VARCHAR)") - .expect("create table"); - engine - .execute("INSERT INTO temp_results VALUES (1, 'a@example.com'), (2, NULL)") - .expect("insert rows"); - + let df = df![ + "id" => [Some(1i64), Some(2)], + "email" => [Some("a@example.com"), None::<&str>] + ] + .unwrap(); let path = std::env::temp_dir().join(format!( "dtoo-profile-{}.json", std::time::SystemTime::now() @@ -476,9 +745,8 @@ mod tests { .expect("clock after epoch") .as_nanos() )); - Profiler::generate( - &engine, + &df, &ProfileOptions { path: path.clone(), format: ProfileFormat::Json, @@ -486,7 +754,6 @@ mod tests { }, ) .expect("generate profile"); - let contents = fs::read_to_string(&path).expect("read profile"); assert!(contents.contains("\"row_count\": 2")); assert!(contents.contains("\"columns\"")); @@ -495,10 +762,10 @@ mod tests { #[test] fn generates_html_profile_file() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (id INTEGER)") - .expect("create table"); + let df = df![ + "id" => [1i64, 2, 3] + ] + .unwrap(); let path = std::env::temp_dir().join(format!( "dtoo-profile-{}.html", std::time::SystemTime::now() @@ -507,7 +774,7 @@ mod tests { .as_nanos() )); Profiler::generate( - &engine, + &df, &ProfileOptions { path: path.clone(), format: ProfileFormat::Html, @@ -553,4 +820,64 @@ mod tests { assert!(csv.contains("\"a,b\"")); assert!(csv.contains("\"a\"\"b\"")); } + + #[test] + fn make_pattern_replaces_digits_and_letters() { + assert_eq!(make_pattern("abc123def"), "aaaNaaa"); + assert_eq!(make_pattern("a@b.com"), "a@a.aaa"); + assert_eq!(make_pattern("2024-01-15"), "N-N-N"); + assert_eq!(make_pattern("hello"), "aaaaa"); + assert_eq!(make_pattern("42"), "N"); + } + + #[test] + fn profile_numeric_column_fills_stats() { + let df = df!["val" => [1.0f64, 2.0, 3.0, 4.0, 5.0]].unwrap(); + let path = std::env::temp_dir().join(format!( + "dtoo-profile-numeric-{}.json", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + Profiler::generate( + &df, + &ProfileOptions { + path: path.clone(), + format: ProfileFormat::Json, + sample_percentage: 100, + }, + ) + .unwrap(); + let contents = fs::read_to_string(&path).unwrap(); + assert!(contents.contains("\"mean\"")); + assert!(contents.contains("\"stddev\"")); + assert!(contents.contains("\"median\"")); + fs::remove_file(path).ok(); + } + + #[test] + fn profile_string_column_fills_lengths_and_patterns() { + let df = df!["email" => ["a@example.com", "b@test.org", "c@foo.net"]].unwrap(); + let path = std::env::temp_dir().join(format!( + "dtoo-profile-str-{}.json", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + Profiler::generate( + &df, + &ProfileOptions { + path: path.clone(), + format: ProfileFormat::Json, + sample_percentage: 100, + }, + ) + .unwrap(); + let contents = fs::read_to_string(&path).unwrap(); + assert!(contents.contains("\"min_length\"")); + assert!(contents.contains("\"pattern_sample\"")); + fs::remove_file(path).ok(); + } } diff --git a/src/query_pipeline.rs b/src/query_pipeline.rs index 14156dd..acff4f1 100644 --- a/src/query_pipeline.rs +++ b/src/query_pipeline.rs @@ -579,7 +579,7 @@ impl<'a> QueryPipelineRunner<'a> { profile_path.display(), profile_format_to_str(self.args.profile_format) )); - Profiler::generate( + Profiler::generate_from_engine( engine, &ProfileOptions { path: profile_path.clone(), From 8057e700a84cd73ea1ea7e98ca02862bb79c139c Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sun, 7 Jun 2026 00:58:25 +0100 Subject: [PATCH 28/30] feat: crypto decrypt/encrypt/discover operate on Polars DataFrames Add discover_wrapped_values_df, decrypt_dataframe, and encrypt_dataframe alongside the existing engine-based functions. The new DataFrame-based functions reuse all per-value crypto routines (CryptoScheme trait impls, detect_wrapped_value, strip_wrapper, wrap_inner) verbatim; only the column-iteration and column-write paths change from DuckDB SQL to Polars StringChunked/df.replace. Engine-based functions and enforce_output_safety are unchanged. 14 new tests cover both schemes, failure modes, and edge cases. Co-Authored-By: Claude Sonnet 4.6 --- src/crypto.rs | 566 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 566 insertions(+) diff --git a/src/crypto.rs b/src/crypto.rs index 277793c..674c1d6 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -7,6 +7,7 @@ use aes_gcm::{ }; use base64::{Engine as _, engine::general_purpose::STANDARD}; use ecb::cipher::{BlockDecryptMut, BlockEncryptMut, block_padding::Pkcs7}; +use polars::prelude::*; use serde::{Deserialize, Serialize}; use sha1::{Digest, Sha1}; @@ -695,6 +696,242 @@ fn decode_32b_key_base64(key_material: &str) -> Result<[u8; 32], DtooError> { Ok(out) } +// --------------------------------------------------------------------------- +// DataFrame-based API (Polars) +// --------------------------------------------------------------------------- + +/// Returns the names of all `DataType::String` columns in `df`. +/// Mirrors what `string_columns(engine)` does for the DuckDB `temp_results` +/// table: only VARCHAR/TEXT/STRING columns are eligible for crypto operations. +fn string_columns_df(df: &DataFrame) -> Vec { + df.get_column_names() + .into_iter() + .filter(|name| matches!(df.column(name).map(|s| s.dtype()), Ok(DataType::String))) + .map(|name| name.to_string()) + .collect() +} + +/// Decides which columns to operate on for a DataFrame, mirroring +/// `detection_columns(engine, mode, scoped_columns)`. +fn detection_columns_df( + df: &DataFrame, + mode: DetectionMode, + scoped_columns: &[String], +) -> Vec { + let all = string_columns_df(df); + let all_set: HashSet<_> = all.iter().cloned().collect(); + + let from_scoped: Vec = scoped_columns + .iter() + .filter(|c| all_set.contains(*c)) + .cloned() + .collect(); + + match mode { + DetectionMode::Auto => all, + DetectionMode::Columns => from_scoped, + DetectionMode::AutoOrColumns => { + if from_scoped.is_empty() { + all + } else { + from_scoped + } + } + } +} + +/// Collect all non-null distinct string values for a column as owned `String`s. +/// Mirrors `select_non_null_column_values(engine, col)`. +fn column_non_null_values_df(df: &DataFrame, column: &str) -> Result, DtooError> { + let series = df.column(column).map_err(|e| DtooError::Config { + message: format!("column `{column}` not found in DataFrame: {e}"), + })?; + let ca = series + .cast(&DataType::String) + .map_err(|e| DtooError::Config { + message: format!("failed to cast column `{column}` to String: {e}"), + })?; + let str_ca = ca.str().map_err(|e| DtooError::Config { + message: format!("failed to access string column `{column}`: {e}"), + })?; + let values: Vec = str_ca.iter().flatten().map(|s| s.to_string()).collect(); + Ok(values) +} + +/// Replace every occurrence of `from_value` with `to_value` in the named column, +/// mutating `df` in place. Mirrors `update_column_value(engine, col, from, to)`. +fn replace_column_value_df( + df: &mut DataFrame, + column: &str, + from_value: &str, + to_value: &str, +) -> Result<(), DtooError> { + let series = df.column(column).map_err(|e| DtooError::Config { + message: format!("column `{column}` not found: {e}"), + })?; + let ca = series.str().map_err(|e| DtooError::Config { + message: format!("column `{column}` is not a String column: {e}"), + })?; + let new_ca: StringChunked = ca + .iter() + .map(|opt| match opt { + Some(v) if v == from_value => Some(to_value), + other => other, + }) + .collect(); + df.replace(column, new_ca.with_name(column.into()).into_column()) + .map(|_| ()) + .map_err(|e| DtooError::Config { + message: format!("failed to replace column `{column}` in DataFrame: {e}"), + }) +} + +/// DataFrame-based equivalent of `discover_wrapped_values`. +/// +/// Scans the String-typed columns of `df` (filtered by `detection.mode` and +/// `scoped_columns`) and returns per-column discovery counts. The per-value +/// detection logic (`detect_wrapped_value`) is reused unchanged. +pub fn discover_wrapped_values_df( + df: &DataFrame, + detection: &DetectionConfig, + scoped_columns: &[String], +) -> Result, DtooError> { + let columns = detection_columns_df(df, detection.mode, scoped_columns); + let mut rows = Vec::with_capacity(columns.len()); + + for column in columns { + let values = column_non_null_values_df(df, &column)?; + let total = values.len(); + let encrypted = values + .iter() + .filter(|value| detect_wrapped_value(value, detection)) + .count(); + rows.push(CryptoDiscoverRow { + column, + total, + encrypted, + }); + } + + Ok(rows) +} + +/// DataFrame-based equivalent of `decrypt_temp_results`. +/// +/// Decrypts every detected-wrapped value in the eligible String columns of `df` +/// and returns the updated DataFrame together with a `CryptoProcessResult` +/// listing which columns were modified. The per-value decrypt logic +/// (`CryptoScheme::decrypt_inner`, `detect_wrapped_value`, `strip_wrapper`, +/// `FailureMode`) is reused unchanged. +pub fn decrypt_dataframe( + mut df: DataFrame, + profile: &CryptoProfile, +) -> Result<(DataFrame, CryptoProcessResult), DtooError> { + validate_detection_config(&profile.detection)?; + + if profile.scheme.scheme_type == SchemeType::LegacyAes128EcbSha1 { + eprintln!( + "Warning: legacy_aes128_ecb_sha1 uses AES-ECB with no integrity protection. Use aes256_gcm_base64 for new data." + ); + } + + let columns = detection_columns_df(&df, profile.detection.mode, &profile.columns); + let scheme = scheme_impl(profile.scheme.scheme_type); + let key_material = load_key_material(profile)?; + + let mut decrypted_columns = Vec::new(); + + for column in columns { + let values = column_non_null_values_df(&df, &column)?; + let mut changed = false; + + for original in values { + if !detect_wrapped_value(&original, &profile.detection) { + continue; + } + let Some(inner) = strip_wrapper(&original, &profile.detection) else { + continue; + }; + if !scheme.detect_inner(inner) { + continue; + } + + match scheme.decrypt_inner(inner, &key_material) { + Ok(plaintext) => { + if plaintext != original { + replace_column_value_df(&mut df, &column, &original, &plaintext)?; + changed = true; + } + } + Err(err) => { + if profile.failure_mode == FailureMode::Strict { + return Err(DtooError::Config { + message: format!( + "decryption failed for column `{column}` using profile `{}`: {err}", + profile.name + ), + }); + } + eprintln!( + "Warning: decryption failed in column `{column}` using profile `{}`: {err}", + profile.name + ); + } + } + } + + if changed { + decrypted_columns.push(column); + } + } + + Ok((df, CryptoProcessResult { decrypted_columns })) +} + +/// DataFrame-based equivalent of `encrypt_columns`. +/// +/// Encrypts every non-wrapped plaintext value in the specified `columns` of +/// `df` and returns the updated DataFrame. The per-value encrypt logic +/// (`CryptoScheme::encrypt_inner`, `wrap_inner`, `detect_wrapped_value`) is +/// reused unchanged. +pub fn encrypt_dataframe( + mut df: DataFrame, + profile: &CryptoProfile, + columns: &[String], +) -> Result { + if columns.is_empty() { + return Ok(df); + } + + validate_detection_config(&profile.detection)?; + let available = string_columns_df(&df); + let available_set: HashSet<_> = available.iter().cloned().collect(); + for column in columns { + if !available_set.contains(column) { + return Err(DtooError::Config { + message: format!("encrypt column `{column}` not found or not string-like"), + }); + } + } + + let scheme = scheme_impl(profile.scheme.scheme_type); + let key_material = load_key_material(profile)?; + + for column in columns { + let values = column_non_null_values_df(&df, column)?; + for plaintext in values { + if detect_wrapped_value(&plaintext, &profile.detection) { + continue; + } + let inner = scheme.encrypt_inner(&plaintext, &key_material)?; + let wrapped = wrap_inner(&inner, &profile.detection); + replace_column_value_df(&mut df, column, &plaintext, &wrapped)?; + } + } + + Ok(df) +} + #[cfg(test)] mod tests { use super::*; @@ -739,4 +976,333 @@ mod tests { enforce_output_safety(&["email".to_string()], true, false).expect("explicit allow"); enforce_output_safety(&["email".to_string()], false, true).expect("reencrypt path"); } + + // ----------------------------------------------------------------------- + // DataFrame-based tests + // ----------------------------------------------------------------------- + + /// Build a `CryptoProfile` for the legacy scheme using a test key set via + /// environment variable. The env var name is unique to these tests so it + /// does not collide with other tests. + fn legacy_test_profile() -> CryptoProfile { + let key = "test-legacy-key"; + // SAFETY: single-threaded test environment; no concurrent env reads. + unsafe { std::env::set_var("DTOO_TEST_LEGACY_KEY", key) }; + CryptoProfile { + name: "test_legacy".to_string(), + detection: DetectionConfig::default(), + scheme: SchemeConfig { + scheme_type: SchemeType::LegacyAes128EcbSha1, + key_env: "DTOO_TEST_LEGACY_KEY".to_string(), + }, + columns: vec![], + failure_mode: FailureMode::BestEffort, + output: OutputSafety { + allow_plaintext: true, + }, + } + } + + /// Build a `CryptoProfile` for the modern AES-256-GCM scheme. + fn modern_test_profile() -> CryptoProfile { + let key_b64 = STANDARD.encode([0xABu8; 32]); + // SAFETY: single-threaded test environment; no concurrent env reads. + unsafe { std::env::set_var("DTOO_TEST_MODERN_KEY", &key_b64) }; + CryptoProfile { + name: "test_modern".to_string(), + detection: DetectionConfig::default(), + scheme: SchemeConfig { + scheme_type: SchemeType::Aes256GcmBase64, + key_env: "DTOO_TEST_MODERN_KEY".to_string(), + }, + columns: vec![], + failure_mode: FailureMode::BestEffort, + output: OutputSafety { + allow_plaintext: true, + }, + } + } + + #[test] + fn df_string_columns_only_string_dtype() { + let df = df![ + "name" => ["alice", "bob"], + "age" => [30i32, 25], + ] + .unwrap(); + let cols = string_columns_df(&df); + assert_eq!(cols, vec!["name".to_string()]); + } + + #[test] + fn df_column_non_null_values_filters_nulls() { + let df = df![ + "col" => [Some("a"), None, Some("b")] + ] + .unwrap(); + let vals = column_non_null_values_df(&df, "col").unwrap(); + assert_eq!(vals, vec!["a".to_string(), "b".to_string()]); + } + + #[test] + fn df_replace_column_value_replaces_matching() { + let mut df = df!["col" => ["hello", "world", "hello"]].unwrap(); + replace_column_value_df(&mut df, "col", "hello", "HELLO").unwrap(); + let vals: Vec> = df.column("col").unwrap().str().unwrap().iter().collect(); + assert_eq!(vals, vec![Some("HELLO"), Some("world"), Some("HELLO")]); + } + + #[test] + fn df_discover_wrapped_values_counts_correctly() { + let profile = legacy_test_profile(); + let scheme = LegacyAes128EcbSha1Scheme; + let key = "test-legacy-key"; + let inner = scheme.encrypt_inner("secret", key).unwrap(); + let wrapped = wrap_inner(&inner, &profile.detection); + + let df = df![ + "email" => [wrapped.as_str(), "plain@example.com", "other-plain"], + ] + .unwrap(); + + let rows = discover_wrapped_values_df(&df, &profile.detection, &[]).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].column, "email"); + assert_eq!(rows[0].total, 3); + assert_eq!(rows[0].encrypted, 1); + } + + #[test] + fn df_discover_scoped_columns_mode() { + let profile = modern_test_profile(); + let scheme = Aes256GcmBase64Scheme; + let key_b64 = STANDARD.encode([0xABu8; 32]); + let inner = scheme.encrypt_inner("data", &key_b64).unwrap(); + let wrapped = wrap_inner(&inner, &profile.detection); + + let df = df![ + "col_a" => [wrapped.as_str(), "plain"], + "col_b" => ["also_plain", "also_plain2"], + ] + .unwrap(); + + // Mode::Columns with scoped_columns=["col_a"] — only col_a examined + let scoped = vec!["col_a".to_string()]; + let detection = DetectionConfig { + mode: DetectionMode::Columns, + ..DetectionConfig::default() + }; + let rows = discover_wrapped_values_df(&df, &detection, &scoped).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].column, "col_a"); + assert_eq!(rows[0].encrypted, 1); + } + + #[test] + fn df_decrypt_dataframe_legacy_roundtrip() { + let profile = legacy_test_profile(); + let scheme = LegacyAes128EcbSha1Scheme; + let key = "test-legacy-key"; + let inner = scheme.encrypt_inner("alice@example.com", key).unwrap(); + let wrapped = wrap_inner(&inner, &profile.detection); + + let df = df![ + "email" => [wrapped.as_str(), "plain@example.com"], + ] + .unwrap(); + + let (df_out, result) = decrypt_dataframe(df, &profile).unwrap(); + assert_eq!(result.decrypted_columns, vec!["email".to_string()]); + + let vals: Vec> = df_out + .column("email") + .unwrap() + .str() + .unwrap() + .iter() + .collect(); + assert_eq!( + vals, + vec![Some("alice@example.com"), Some("plain@example.com")] + ); + } + + #[test] + fn df_decrypt_dataframe_modern_roundtrip() { + let profile = modern_test_profile(); + let scheme = Aes256GcmBase64Scheme; + let key_b64 = STANDARD.encode([0xABu8; 32]); + let inner = scheme.encrypt_inner("secret-data", &key_b64).unwrap(); + let wrapped = wrap_inner(&inner, &profile.detection); + + let df = df![ + "payload" => [wrapped.as_str(), "unchanged"], + ] + .unwrap(); + + let (df_out, result) = decrypt_dataframe(df, &profile).unwrap(); + assert_eq!(result.decrypted_columns, vec!["payload".to_string()]); + + let vals: Vec> = df_out + .column("payload") + .unwrap() + .str() + .unwrap() + .iter() + .collect(); + assert_eq!(vals, vec![Some("secret-data"), Some("unchanged")]); + } + + #[test] + fn df_decrypt_no_change_when_no_wrapped_values() { + let profile = legacy_test_profile(); + let df = df!["col" => ["plain", "also_plain"]].unwrap(); + let (_, result) = decrypt_dataframe(df, &profile).unwrap(); + assert!( + result.decrypted_columns.is_empty(), + "no columns should be listed as decrypted" + ); + } + + #[test] + fn df_decrypt_strict_mode_errors_on_bad_ciphertext() { + // Build a profile with strict failure mode + let key = "strict-key"; + // SAFETY: single-threaded test environment; no concurrent env reads. + unsafe { std::env::set_var("DTOO_TEST_STRICT_KEY", key) }; + let profile = CryptoProfile { + name: "strict".to_string(), + detection: DetectionConfig::default(), + scheme: SchemeConfig { + scheme_type: SchemeType::LegacyAes128EcbSha1, + key_env: "DTOO_TEST_STRICT_KEY".to_string(), + }, + columns: vec![], + failure_mode: FailureMode::Strict, + output: OutputSafety { + allow_plaintext: true, + }, + }; + + // Manufacture a value that looks wrapped but has garbage ciphertext + // (valid base64 but cannot be AES-decrypted cleanly under any key). + // 16 bytes of zeros will decrypt to something but PKCS7 unpadding + // will almost certainly fail because the decrypted bytes are unlikely + // to have a valid pad byte (0x10 = 16 repeated 16 times). + // Use all-0xFF bytes: when decrypted under "strict-key", the result + // will not have valid PKCS7 padding (pad byte 0xFF means 255 padding + // bytes, which is impossible in a 16-byte block). + let garbage_inner = STANDARD.encode([0xFFu8; 32]); + let wrapped = format!( + "{}{}{}", + profile.detection.start_marker, garbage_inner, profile.detection.end_marker + ); + + let df = df!["col" => [wrapped.as_str()]].unwrap(); + let err = decrypt_dataframe(df, &profile).unwrap_err(); + assert!(err.to_string().contains("decryption failed")); + } + + #[test] + fn df_encrypt_dataframe_legacy_roundtrip() { + let profile = legacy_test_profile(); + let df = df!["email" => ["alice@example.com", "bob@example.com"]].unwrap(); + let columns = vec!["email".to_string()]; + + let df_enc = encrypt_dataframe(df, &profile, &columns).unwrap(); + + // Every value should now be wrapped + let detection = &profile.detection; + let vals: Vec> = df_enc + .column("email") + .unwrap() + .str() + .unwrap() + .iter() + .collect(); + for val in &vals { + let v = val.unwrap(); + assert!( + detect_wrapped_value(v, detection), + "expected wrapped value, got: {v}" + ); + } + + // Decrypt them back and confirm round-trip + let (df_dec, result) = decrypt_dataframe(df_enc, &profile).unwrap(); + assert_eq!(result.decrypted_columns, vec!["email".to_string()]); + let decrypted: Vec> = df_dec + .column("email") + .unwrap() + .str() + .unwrap() + .iter() + .collect(); + assert_eq!( + decrypted, + vec![Some("alice@example.com"), Some("bob@example.com")] + ); + } + + #[test] + fn df_encrypt_dataframe_modern_roundtrip() { + let profile = modern_test_profile(); + let df = df!["secret" => ["value1", "value2"]].unwrap(); + let columns = vec!["secret".to_string()]; + + let df_enc = encrypt_dataframe(df, &profile, &columns).unwrap(); + let (df_dec, result) = decrypt_dataframe(df_enc, &profile).unwrap(); + assert_eq!(result.decrypted_columns, vec!["secret".to_string()]); + let vals: Vec> = df_dec + .column("secret") + .unwrap() + .str() + .unwrap() + .iter() + .collect(); + assert_eq!(vals, vec![Some("value1"), Some("value2")]); + } + + #[test] + fn df_encrypt_skips_already_wrapped_values() { + let profile = legacy_test_profile(); + let scheme = LegacyAes128EcbSha1Scheme; + let key = "test-legacy-key"; + let inner = scheme.encrypt_inner("already", key).unwrap(); + let wrapped = wrap_inner(&inner, &profile.detection); + let wrapped_clone = wrapped.clone(); + + let df = df!["col" => [wrapped.as_str(), "plaintext"]].unwrap(); + let columns = vec!["col".to_string()]; + let df_enc = encrypt_dataframe(df, &profile, &columns).unwrap(); + + let vals: Vec> = df_enc + .column("col") + .unwrap() + .str() + .unwrap() + .iter() + .collect(); + // The already-wrapped value should be unchanged (same bytes) + assert_eq!(vals[0], Some(wrapped_clone.as_str())); + // The plaintext should now be wrapped + assert!(detect_wrapped_value(vals[1].unwrap(), &profile.detection)); + } + + #[test] + fn df_encrypt_unknown_column_errors() { + let profile = legacy_test_profile(); + let df = df!["col" => ["value"]].unwrap(); + let err = encrypt_dataframe(df, &profile, &["nonexistent".to_string()]).unwrap_err(); + assert!(err.to_string().contains("not found or not string-like")); + } + + #[test] + fn df_encrypt_empty_columns_is_noop() { + let profile = legacy_test_profile(); + let df = df!["col" => ["value"]].unwrap(); + let df_out = encrypt_dataframe(df.clone(), &profile, &[]).unwrap(); + // shape unchanged + assert_eq!(df_out.shape(), df.shape()); + } } From 3233caaed2ffa11955d7e881b3f4adc9098b3af9 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sun, 7 Jun 2026 01:15:24 +0100 Subject: [PATCH 29/30] fix: profiler distinct_count excludes nulls; schema match case-insensitive; DECIMAL default (18,3) Co-Authored-By: Claude Sonnet 4.6 --- src/profiler.rs | 18 ++++++++++++++++- src/schema.rs | 53 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/src/profiler.rs b/src/profiler.rs index e9561cb..09d74d7 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -152,7 +152,11 @@ fn profile_column(series: &Column, total_rows: usize) -> Result [Some("a"), Some("a"), None::<&str>]].unwrap(); + let report = build_report(&df, 100).expect("build report"); + assert_eq!( + report.columns[0].distinct_count, 1, + "distinct_count must exclude NULLs (parity with DuckDB COUNT(DISTINCT))" + ); + } + #[test] fn make_pattern_replaces_digits_and_letters() { assert_eq!(make_pattern("abc123def"), "aaaNaaa"); diff --git a/src/schema.rs b/src/schema.rs index 8111b69..cf95fe8 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -118,8 +118,8 @@ pub fn duckdb_type_to_polars(data_type: &str) -> Result { "TIMESTAMP" | "DATETIME" => DataType::Datetime(TimeUnit::Microseconds, None), "DECIMAL" | "NUMERIC" => { let (p, s) = parse_decimal_params(&upper); - // Polars 0.54 Decimal(usize, usize): default precision=38, scale=0 - DataType::Decimal(p.unwrap_or(38), s.unwrap_or(0)) + // DuckDB bare DECIMAL defaults to DECIMAL(18,3); explicit params override. + DataType::Decimal(p.unwrap_or(18), s.unwrap_or(3)) } _ => { return Err(DtooError::Schema { @@ -154,16 +154,19 @@ pub fn coerce_to_schema(lf: LazyFrame, columns: &[SchemaColumn]) -> Result = schema + // Build a map from lowercased source column name → actual source column name + // so that declared columns match source columns case-insensitively (parity + // with `projected_query_for_schema` which lowercases both sides). + let lower_to_actual: HashMap = schema .iter_names() - .map(|n: &PlSmallStr| n.to_string()) + .map(|n: &PlSmallStr| (n.to_ascii_lowercase(), n.to_string())) .collect(); let mut exprs: Vec = Vec::with_capacity(columns.len()); for c in columns { let dt = duckdb_type_to_polars(&c.data_type)?; - let e = if present.contains(&c.name) { - col(c.name.as_str()).cast(dt).alias(c.name.as_str()) + let e = if let Some(actual) = lower_to_actual.get(&c.name.to_ascii_lowercase()) { + col(actual.as_str()).cast(dt).alias(c.name.as_str()) } else { lit(LiteralValue::untyped_null()) .cast(dt) @@ -472,6 +475,44 @@ mod tests { assert_eq!(out.column("name").unwrap().null_count(), 1); // missing -> null } + #[test] + fn coerce_matches_source_columns_case_insensitively() { + // Source has "Name" (capital N); declared schema uses "name" (lowercase). + // Must match case-insensitively and produce "alice", not NULL. + use crate::types::SchemaColumn; + use polars::prelude::*; + let lf = df!["Name" => ["alice"]].unwrap().lazy(); + let cols = vec![SchemaColumn { + name: "name".into(), + data_type: "VARCHAR".into(), + }]; + let out = coerce_to_schema(lf, &cols).unwrap().collect().unwrap(); + let col_name = out.column("name").expect("output column 'name' must exist"); + assert_eq!(col_name.null_count(), 0, "matched column must not be null"); + assert_eq!( + col_name.str().unwrap().get(0).expect("first value"), + "alice", + "case-insensitive match must preserve source value" + ); + } + + #[test] + fn decimal_bare_defaults_to_18_3() { + use polars::prelude::*; + // Bare DECIMAL (no params) must default to DuckDB's DECIMAL(18,3). + assert_eq!( + duckdb_type_to_polars("DECIMAL").unwrap(), + DataType::Decimal(18, 3), + "bare DECIMAL must map to Decimal(18,3)" + ); + // Explicit DECIMAL(10,2) must not be overridden. + assert_eq!( + duckdb_type_to_polars("DECIMAL(10,2)").unwrap(), + DataType::Decimal(10, 2), + "explicit DECIMAL(10,2) must map to Decimal(10,2)" + ); + } + fn temp_schema(contents: &str) -> std::path::PathBuf { let path = std::env::temp_dir().join(format!("dtoo-schema-{}.yaml", unique_suffix())); fs::write(&path, contents).expect("write schema file"); From d837761568d86596664bbf5eb796d146d6c3e533 Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sun, 7 Jun 2026 01:17:54 +0100 Subject: [PATCH 30/30] feat: output writer emits a Polars DataFrame via PolarsEngine Add DataFrame-based write/write_and_get_destination methods on OutputWriter that delegate to PolarsEngine::write. Rename the existing DuckDB-backed methods to write_from_engine / write_and_get_destination_from_engine so the query pipeline build stays green until P2-9 switches the pipeline to the DataFrame path; the _from_engine variants will be deleted in P2-12. Port the three engine tests (write_uses_adjusted_output_path, errors_when_output_directory_missing, returns_effective_destination_path) to use PolarsEngine + df![]; keep the legacy DuckDB tests alongside them under _from_engine names. Co-Authored-By: Claude Sonnet 4.6 --- src/output_writer.rs | 196 ++++++++++++++++++++++++++++++++++++++---- src/query_pipeline.rs | 2 +- 2 files changed, 178 insertions(+), 20 deletions(-) diff --git a/src/output_writer.rs b/src/output_writer.rs index 76e98b5..9de96c1 100644 --- a/src/output_writer.rs +++ b/src/output_writer.rs @@ -1,8 +1,11 @@ use std::path::{Path, PathBuf}; +use polars::prelude::DataFrame; + use crate::{ engine::DuckDbEngine, error::DtooError, + polars_engine::PolarsEngine, types::{CompressionCodec, ExportFormat}, }; @@ -27,8 +30,76 @@ impl OutputWriter { Self { config } } - /// Export temp_results to configured destination. - pub fn write(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { + /// Export a Polars DataFrame to configured destination via `PolarsEngine`. + pub fn write(&self, engine: &PolarsEngine, df: DataFrame) -> Result<(), DtooError> { + let destination = self.effective_destination_path(); + let compression = self.effective_compression(&destination); + if let Some(path) = &destination + && let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + && !parent.exists() + { + return Err(DtooError::Output { + message: format!("output directory does not exist: {}", parent.display()), + }); + } + + engine.write( + df, + destination.as_deref(), + self.config.format, + self.config.header, + self.config.delimiter, + compression, + )?; + + if let (Some(original), Some(resolved)) = (&self.config.output, &destination) + && original != resolved + { + eprintln!("Info: output path adjusted to {}", resolved.display()); + } + + Ok(()) + } + + /// Write a Polars DataFrame and return effective destination path (`None` when stdout). + pub fn write_and_get_destination( + &self, + engine: &PolarsEngine, + df: DataFrame, + ) -> Result, DtooError> { + let destination = self.effective_destination_path(); + let compression = self.effective_compression(&destination); + if let Some(path) = &destination + && let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + && !parent.exists() + { + return Err(DtooError::Output { + message: format!("output directory does not exist: {}", parent.display()), + }); + } + engine.write( + df, + destination.as_deref(), + self.config.format, + self.config.header, + self.config.delimiter, + compression, + )?; + if let (Some(original), Some(resolved)) = (&self.config.output, &destination) + && original != resolved + { + eprintln!("Info: output path adjusted to {}", resolved.display()); + } + Ok(destination) + } + + /// Export temp_results to configured destination via `DuckDbEngine`. + /// + /// Kept for the DuckDB-based pipeline until P2-9 switches to the DataFrame path. + /// Will be removed in P2-12. + pub fn write_from_engine(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { let destination = self.effective_destination_path(); let compression = self.effective_compression(&destination); if let Some(path) = &destination @@ -58,8 +129,11 @@ impl OutputWriter { Ok(()) } - /// Write output and return effective destination path (`None` when stdout). - pub fn write_and_get_destination( + /// Write via `DuckDbEngine` and return effective destination path (`None` when stdout). + /// + /// Kept for the DuckDB-based pipeline until P2-9 switches to the DataFrame path. + /// Will be removed in P2-12. + pub fn write_and_get_destination_from_engine( &self, engine: &DuckDbEngine, ) -> Result, DtooError> { @@ -141,6 +215,7 @@ fn adjust_compressed_extension( mod tests { use super::*; use crate::engine::{CloudSettings, EngineConfig}; + use polars::df; use std::fs; fn test_engine() -> DuckDbEngine { @@ -194,13 +269,8 @@ mod tests { #[test] fn write_uses_adjusted_output_path() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (id INTEGER)") - .expect("create table should work"); - engine - .execute("INSERT INTO temp_results VALUES (1)") - .expect("insert should work"); + let engine = PolarsEngine::new(); + let df = df!["id" => [1i64]].expect("df macro should work"); let base = std::env::temp_dir().join(format!( "dtoo-output-writer-{}", @@ -217,7 +287,7 @@ mod tests { delimiter: ',', compression: Some(CompressionCodec::Gzip), }); - writer.write(&engine).expect("write should succeed"); + writer.write(&engine, df).expect("write should succeed"); let adjusted = PathBuf::from(format!("{}.gz", base.to_string_lossy())); assert!(adjusted.exists()); @@ -227,10 +297,8 @@ mod tests { #[test] fn errors_when_output_directory_missing() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (id INTEGER)") - .expect("create table should work"); + let engine = PolarsEngine::new(); + let df = df!["id" => [1i64]].expect("df macro should work"); let writer = OutputWriter::new(OutputWriterConfig { output: Some(PathBuf::from("/tmp/dtoo-nonexistent-subdir-12345/out.csv")), @@ -240,7 +308,7 @@ mod tests { compression: None, }); let err = writer - .write(&engine) + .write(&engine, df) .expect_err("should fail for missing dir"); assert!(matches!(err, DtooError::Output { .. })); } @@ -260,6 +328,39 @@ mod tests { #[test] fn returns_effective_destination_path() { + let engine = PolarsEngine::new(); + let df = df!["id" => [1i64]].expect("df macro should work"); + + let base = std::env::temp_dir().join(format!( + "dtoo-output-writer-path-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock after epoch") + .as_nanos() + )); + let writer = OutputWriter::new(OutputWriterConfig { + output: Some(base.clone()), + format: ExportFormat::Csv, + header: true, + delimiter: ',', + compression: Some(CompressionCodec::Gzip), + }); + let destination = writer + .write_and_get_destination(&engine, df) + .expect("write should succeed"); + assert_eq!( + destination, + Some(PathBuf::from(format!("{}.gz", base.to_string_lossy()))) + ); + std::fs::remove_file(format!("{}.gz", base.to_string_lossy())).ok(); + } + + // ----------------------------------------------------------------------- + // Legacy DuckDB-backed tests (kept until P2-9 pipeline switch) + // ----------------------------------------------------------------------- + + #[test] + fn write_from_engine_uses_adjusted_output_path() { let engine = test_engine(); engine .execute("CREATE TABLE temp_results (id INTEGER)") @@ -267,8 +368,65 @@ mod tests { engine .execute("INSERT INTO temp_results VALUES (1)") .expect("insert should work"); + let base = std::env::temp_dir().join(format!( - "dtoo-output-writer-path-{}", + "dtoo-output-writer-legacy-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock after epoch") + .as_nanos() + )); + + let writer = OutputWriter::new(OutputWriterConfig { + output: Some(base.clone()), + format: ExportFormat::Csv, + header: true, + delimiter: ',', + compression: Some(CompressionCodec::Gzip), + }); + writer + .write_from_engine(&engine) + .expect("write should succeed"); + + let adjusted = PathBuf::from(format!("{}.gz", base.to_string_lossy())); + assert!(adjusted.exists()); + + fs::remove_file(adjusted).ok(); + } + + #[test] + fn write_from_engine_errors_when_output_directory_missing() { + let engine = test_engine(); + engine + .execute("CREATE TABLE temp_results (id INTEGER)") + .expect("create table should work"); + + let writer = OutputWriter::new(OutputWriterConfig { + output: Some(PathBuf::from( + "/tmp/dtoo-nonexistent-subdir-legacy-12345/out.csv", + )), + format: ExportFormat::Csv, + header: true, + delimiter: ',', + compression: None, + }); + let err = writer + .write_from_engine(&engine) + .expect_err("should fail for missing dir"); + assert!(matches!(err, DtooError::Output { .. })); + } + + #[test] + fn write_and_get_destination_from_engine_returns_path() { + let engine = test_engine(); + engine + .execute("CREATE TABLE temp_results (id INTEGER)") + .expect("create table should work"); + engine + .execute("INSERT INTO temp_results VALUES (1)") + .expect("insert should work"); + let base = std::env::temp_dir().join(format!( + "dtoo-output-writer-path-legacy-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("clock after epoch") @@ -282,7 +440,7 @@ mod tests { compression: Some(CompressionCodec::Gzip), }); let destination = writer - .write_and_get_destination(&engine) + .write_and_get_destination_from_engine(&engine) .expect("write should succeed"); assert_eq!( destination, diff --git a/src/query_pipeline.rs b/src/query_pipeline.rs index acff4f1..e8bbb42 100644 --- a/src/query_pipeline.rs +++ b/src/query_pipeline.rs @@ -557,7 +557,7 @@ impl<'a> QueryPipelineRunner<'a> { delimiter: self.args.delimiter, compression: self.args.compress.map(to_compression_codec), }); - let written_output_path = writer.write_and_get_destination(engine)?; + let written_output_path = writer.write_and_get_destination_from_engine(engine)?; self.summary.output_path = written_output_path .as_ref() .map(|path| path.display().to_string());