diff --git a/Cargo.lock b/Cargo.lock index 5fc6656..df76357 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,17 +43,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.12" @@ -61,7 +50,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -172,15 +160,6 @@ dependencies = [ "object", ] -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "argminmax" version = "0.6.3" @@ -209,169 +188,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "arrow" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", -] - -[[package]] -name = "arrow-arith" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "num-traits", -] - -[[package]] -name = "arrow-array" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "half", - "hashbrown 0.16.1", - "num-complex", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-buffer" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" -dependencies = [ - "bytes", - "half", - "num-bigint", - "num-traits", -] - -[[package]] -name = "arrow-cast" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "atoi", - "base64", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num-traits", - "ryu", -] - -[[package]] -name = "arrow-data" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" -dependencies = [ - "arrow-buffer", - "arrow-schema", - "half", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-ord" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", -] - -[[package]] -name = "arrow-row" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "half", -] - -[[package]] -name = "arrow-schema" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" -dependencies = [ - "bitflags", -] - -[[package]] -name = "arrow-select" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" -dependencies = [ - "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "num-traits", -] - -[[package]] -name = "arrow-string" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "memchr", - "num-traits", - "regex", - "regex-syntax", -] - [[package]] name = "async-channel" version = "2.5.0" @@ -403,7 +219,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -414,16 +230,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", + "syn", ] [[package]] @@ -482,18 +289,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "blake3" version = "1.8.5" @@ -526,30 +321,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "borsh" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" -dependencies = [ - "borsh-derive", - "bytes", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" -dependencies = [ - "once_cell", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "boxcar" version = "0.2.14" @@ -583,28 +354,6 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bytemuck" version = "1.25.0" @@ -622,7 +371,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -654,15 +403,9 @@ dependencies = [ "log", "quick-xml", "serde", - "zip 7.2.0", + "zip", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "castaway" version = "0.2.4" @@ -772,7 +515,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -844,26 +587,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "tiny-keccak", -] - [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1011,17 +734,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "digest" version = "0.10.7" @@ -1040,7 +752,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1063,7 +775,6 @@ dependencies = [ "chrono", "clap", "comfy-table", - "duckdb", "ecb", "flate2", "glob", @@ -1081,23 +792,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "duckdb" -version = "1.10501.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13bc6d6487032fc2825a62ef8b4924b2378a2eb3166e132e5f3141ae9dd633f" -dependencies = [ - "arrow", - "cast", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libduckdb-sys", - "num-integer", - "rust_decimal", - "strum", -] - [[package]] name = "dyn-clone" version = "1.0.20" @@ -1177,12 +871,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - [[package]] name = "fallible-streaming-iterator" version = "0.1.9" @@ -1201,17 +889,6 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" -[[package]] -name = "filetime" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" -dependencies = [ - "cfg-if", - "libc", - "libredox", -] - [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1275,12 +952,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" version = "0.3.32" @@ -1337,7 +1008,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1479,15 +1150,6 @@ dependencies = [ "serde", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] - [[package]] name = "hashbrown" version = "0.15.5" @@ -1519,15 +1181,6 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" -[[package]] -name = "hashlink" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "heck" version = "0.5.0" @@ -1630,7 +1283,6 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", ] [[package]] @@ -1900,104 +1552,18 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" -[[package]] -name = "lexical-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" -dependencies = [ - "lexical-parse-integer", - "lexical-util", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "lexical-util" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" - -[[package]] -name = "lexical-write-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" -dependencies = [ - "lexical-util", - "lexical-write-integer", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" -dependencies = [ - "lexical-util", -] - [[package]] name = "libc" version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" -[[package]] -name = "libduckdb-sys" -version = "1.10501.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12096c1694924782b3fe21e790630b77bacb4fcb7ad9d7ee0fec626f985bf248" -dependencies = [ - "cc", - "flate2", - "pkg-config", - "reqwest", - "serde", - "serde_json", - "tar", - "vcpkg", - "zip 6.0.0", -] - [[package]] name = "libm" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "libredox" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" -dependencies = [ - "bitflags", - "libc", - "plain", - "redox_syscall 0.7.4", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2110,25 +1676,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - [[package]] name = "num-derive" version = "0.4.2" @@ -2137,16 +1684,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", + "syn", ] [[package]] @@ -2278,7 +1816,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link 0.2.1", ] @@ -2319,12 +1857,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" -[[package]] -name = "plain" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" - [[package]] name = "planus" version = "1.1.1" @@ -3024,16 +2556,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", -] - -[[package]] -name = "proc-macro-crate" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" -dependencies = [ - "toml_edit", + "syn", ] [[package]] @@ -3055,26 +2578,6 @@ dependencies = [ "cc", ] -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "quick-xml" version = "0.39.4" @@ -3162,30 +2665,13 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] @@ -3200,16 +2686,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -3300,7 +2776,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3312,15 +2788,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_syscall" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" -dependencies = [ - "bitflags", -] - [[package]] name = "ref-cast" version = "1.0.25" @@ -3338,7 +2805,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3370,15 +2837,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck", -] - [[package]] name = "reqwest" version = "0.12.28" @@ -3387,7 +2845,6 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", - "futures-channel", "futures-core", "futures-util", "h2", @@ -3420,7 +2877,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", ] [[package]] @@ -3437,35 +2893,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rkyv" -version = "0.7.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" -dependencies = [ - "bitvec", - "bytecheck", - "bytes", - "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "rmp" version = "0.8.15" @@ -3485,30 +2912,13 @@ dependencies = [ "serde", ] -[[package]] -name = "rust_decimal" -version = "1.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" -dependencies = [ - "arrayvec", - "borsh", - "bytes", - "num-traits", - "rand 0.8.5", - "rkyv", - "serde", - "serde_json", - "wasm-bindgen", -] - [[package]] name = "rust_xlsxwriter" version = "0.95.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f281b687352597d29efaad39701d1167d5c48aa76fb973e392bc13e9d44e7f36" dependencies = [ - "zip 7.2.0", + "zip", ] [[package]] @@ -3613,12 +3023,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" version = "3.7.0" @@ -3675,7 +3079,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3788,7 +3192,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4255126f310d2ba20048db6321c81ab376f6a6735608bf11f0785c41f01f64e3" dependencies = [ - "ahash 0.8.12", + "ahash", "halfbrown", "once_cell", "ref-cast", @@ -3866,7 +3270,7 @@ checksum = "028e551d5e270b31b9f3ea271778d9d827148d4287a5d96167b6bb9787f5cc38" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3921,15 +3325,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" -dependencies = [ - "strum_macros", -] - [[package]] name = "strum_macros" version = "0.27.2" @@ -3939,7 +3334,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3948,17 +3343,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.117" @@ -3987,7 +3371,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4004,23 +3388,6 @@ dependencies = [ "windows", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - -[[package]] -name = "tar" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "thiserror" version = "2.0.18" @@ -4038,7 +3405,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4050,15 +3417,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.8.3" @@ -4107,7 +3465,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4133,36 +3491,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml_datetime" -version = "1.1.1+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" -dependencies = [ - "serde_core", -] - -[[package]] -name = "toml_edit" -version = "0.25.11+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" -dependencies = [ - "indexmap", - "toml_datetime", - "toml_parser", - "winnow", -] - -[[package]] -name = "toml_parser" -version = "1.1.2+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" -dependencies = [ - "winnow", -] - [[package]] name = "tower" version = "0.5.3" @@ -4227,7 +3555,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4375,12 +3703,6 @@ dependencies = [ "ryu", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" @@ -4445,7 +3767,6 @@ dependencies = [ "cfg-if", "once_cell", "rustversion", - "serde", "wasm-bindgen-macro", "wasm-bindgen-shared", ] @@ -4479,7 +3800,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wasm-bindgen-shared", ] @@ -4559,15 +3880,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "winapi" version = "0.3.9" @@ -4666,7 +3978,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4677,7 +3989,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4912,15 +4224,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" -[[package]] -name = "winnow" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" -dependencies = [ - "memchr", -] - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -4951,7 +4254,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn 2.0.117", + "syn", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -4967,7 +4270,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -5015,25 +4318,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - -[[package]] -name = "xattr" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" -dependencies = [ - "libc", - "rustix", -] - [[package]] name = "xxhash-rust" version = "0.8.15" @@ -5059,7 +4343,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -5080,7 +4364,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5100,7 +4384,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -5140,21 +4424,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "zip" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" -dependencies = [ - "arbitrary", - "crc32fast", - "flate2", - "indexmap", - "memchr", - "zopfli", + "syn", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6fdfc21..acd0f77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" thiserror = "2" -duckdb = { version = "1", features = ["bundled"] } glob = "0.3" uuid = { version = "1", features = ["v4"] } sha2 = "0.10" diff --git a/docs/DESIGN.md b/docs/DESIGN.md index c98993a..03cc1d9 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -4,7 +4,18 @@ A Rust CLI tool for data engineers to query, profile, and transform data across ## Core Engine -**DuckDB** (via `duckdb` Rust crate) — provides SQL execution, format readers (Parquet, CSV, NDJSON, Excel), glob support, cloud storage (S3/GCS/Azure via httpfs/azure extensions), temp tables, hashing, and UUID generation. +**Polars** (pure Rust, `polars` crate ~0.54.x) — provides SQL execution via `SQLContext`, format readers (Parquet, CSV/TSV, NDJSON, and Excel via the pure-Rust `calamine` backend), glob resolution, type-safe lazy evaluation, and the `concat_lf_diagonal` union-by-name schema evolution. There is no bundled C++ toolchain and no runtime extension downloads. + +**Cloud storage** (S3/GCS/Azure) is **deferred in this build**: cloud paths (`s3://`, `gs://`, `az://`) return a clear `"cloud storage … is not supported in this build yet"` error. The cloud CLI flags (`--s3-region`, `--s3-profile`, `--gcs-project`, `--azure-account`) still parse for forward compatibility. + +### SQL Surface Limitations + +User-facing SQL (`--where`, `--filter-sql`, `--post-sql`, reference-table JOINs) runs on Polars `SQLContext`. Compared to DuckDB, the SQL surface is narrower: + +- **`DELETE` / `UPDATE` statements are treated as transforms**, not mutations. For example, `DELETE FROM _ WHERE x` drops matching rows and returns a result set rather than raising an error. Do not rely on DML semantics in `--post-sql`. +- **Window functions** (`OVER (PARTITION BY … ORDER BY …)`) have known correctness issues in Polars SQL. Avoid them; dtoo does not attempt to detect or warn about their use. +- **Narrower function library**: some exotic DuckDB date, regex, and string functions are absent. Unsupported SQL returns a clear error (it never silently hangs — which was the motivation for the migration). +- Errors are always explicit `Result` values; the engine never hangs on malformed input. --- @@ -36,9 +47,9 @@ Execution order within `dtoo query`: 1. Resolve file list (glob pattern / pipe input / explicit paths) 2. Apply --exclude patterns to filter file list 3. If --dry-run: display plan and exit -4. Init DuckDB in-memory, load extensions -5. Load reference tables into named DuckDB tables -6. Create temp_results table (schema from first file, or explicit schema) +4. Init Polars engine; scan reference table files into named LazyFrames +5. Scan each input file as a LazyFrame; register refs and _ in SQLContext +6. Create accumulated result (LazyFrame; schema from first file or explicit --schema) 7. For each file: a. Register file as `_` (the magic table name) b. Apply --where clause if specified: SELECT * FROM _ WHERE {where} @@ -173,11 +184,11 @@ dtoo fingerprint Files are resolved from one of three sources (mutually exclusive): -1. **--glob**: DuckDB-compatible glob pattern. Supports `**` for recursive matching. +1. **--glob**: Glob pattern. Supports `**` for recursive matching (resolved by Polars native glob scan). 2. **--pipe file**: Newline-delimited file paths from stdin. 3. **--pipe data**: Raw data stream from stdin (requires `--stdin-format`). -Cloud paths (s3://, gs://, az://) are supported in all modes. +Cloud paths (`s3://`, `gs://`, `az://`) are **not supported in this build** — they return an explicit error. See Core Engine above. Format is auto-detected from file extension: - `.parquet` — Parquet @@ -190,6 +201,8 @@ Format is auto-detected from file extension: 2. **`--sheet` flag**: applies to all `.xlsx` files matched by glob or pipe 3. **Default**: first sheet +**Excel reading behavior (calamine):** Every cell is read as a string; type inference is deferred to downstream SQL casts or explicit `--schema`. A data row wider than the header row is an error (no silent data loss). + Examples: ```bash # All xlsx files, same sheet @@ -205,11 +218,12 @@ find . -name "*.xlsx" | dtoo query --pipe file --sheet "Data" ### Schema Handling -**Default (no --schema):** Union-by-name with type promotion. The temp_results table schema evolves as new columns are encountered. DuckDB handles type promotion (e.g., INT -> BIGINT -> DOUBLE). +**Default (no --schema):** Union-by-name with type promotion. The accumulated result schema evolves as new columns are encountered. Polars `concat_lf_diagonal` handles type promotion (e.g., Int32 → Int64 → Float64) and fills missing columns with `null`. **Explicit (--schema):** Schema file defines the target columns and types. Files are coerced to match. Extra columns in source files are ignored; missing columns become NULL. -Schema file format (YAML): +Schema file format (YAML). Type strings use DuckDB-style names (e.g. `INTEGER`, `VARCHAR`, `DECIMAL(10,2)`, `TIMESTAMP`) which are mapped to Polars dtypes at load time. Bare `DECIMAL` defaults to `DECIMAL(18,3)`. + ```yaml columns: - name: id @@ -226,12 +240,11 @@ columns: When both are specified, `--where` is applied first as a pre-filter, then `--filter-sql` operates on the result: -```sql +``` -- Internal execution when both specified: --- Step 1: Apply --where -CREATE TEMP VIEW _pre AS SELECT * FROM read_parquet('{file}') WHERE {where_clause}; --- Step 2: Apply --filter-sql (user's SQL, _ now points to _pre) -INSERT INTO temp_results SELECT ... FROM _pre ...; +-- Step 1: Scan file → LazyFrame; register as _; execute: SELECT * FROM _ WHERE {where_clause} +-- Step 2: Apply --filter-sql against the filtered LazyFrame (user's SELECT from _) +-- Step 3: Concatenate result into the accumulated LazyFrame (union-by-name) ``` When only `--where` is specified, it's equivalent to `--filter-sql "SELECT * FROM _ WHERE {clause}"`. @@ -240,7 +253,7 @@ When only `--filter-sql` is specified, it runs directly against the file. ### Reference Tables -Loaded once at startup into named DuckDB tables: +Loaded once at startup into named LazyFrames registered in the `SQLContext`: ``` --ref regions=ref/regions.parquet --ref products=lookups/products.csv @@ -281,7 +294,7 @@ NULL values remain NULL (not masked). ### Profiling -Uses DuckDB's analytical capabilities to produce per-column statistics: +Produces per-column statistics using Polars aggregate expressions: | Metric | Applies To | |--------|-----------| @@ -533,22 +546,23 @@ CLI flags override config file values. Config file can be combined with CLI flag | Crate | Purpose | |-------|---------| -| `duckdb` | Core engine, SQL execution, file readers | +| `polars` | Core engine: SQL execution (`SQLContext`), lazy evaluation, format readers (Parquet, CSV, NDJSON) | +| `calamine` | Excel reader (pure Rust; bundled via Polars `excel` feature) | +| `flate2` | gzip output wrapping for CSV/NDJSON (Polars has no native text-output compression) | +| `zstd` | zstd output wrapping for CSV/NDJSON | | `clap` | CLI argument parsing (derive API) | | `serde` + `serde_yaml` | Config file and schema parsing | | `uuid` | Lineage UUID generation | | `sha2` + `hmac` | Column masking and file fingerprinting | | `chrono` | Timestamp handling | -| `glob` | File pattern matching (fallback for non-DuckDB resolution) | | `indicatif` | Progress bars for file processing | | `comfy-table` | Terminal table output for inspect | -| `tokio` | Async runtime (needed for cloud storage operations) | --- ## v1.0 Scope (Open Source) -Everything described above, including cloud storage (S3, GCS, Azure) since DuckDB extensions make this nearly free. +Everything described above. **Cloud storage (S3, GCS, Azure) is deferred** in the current build — cloud paths return an explicit error. Cloud support will be re-enabled in a follow-up once the Polars engine is proven in local-file deployments. --- @@ -560,7 +574,7 @@ Everything described above, including cloud storage (S3, GCS, Azure) since DuckD | Core query/transform | Yes | Yes | Yes | | Profile/inspect/fingerprint | Yes | Yes | Yes | | Lineage, masking, pipe mode | Yes | Yes | Yes | -| Cloud storage (S3/GCS/Azure) | Yes | Yes | Yes | +| Cloud storage (S3/GCS/Azure) | Deferred | Deferred | Deferred | | Config files, manifests | Yes | Yes | Yes | | Data quality assertions | - | Yes | Yes | | Incremental processing | - | Yes | Yes | @@ -639,8 +653,7 @@ overall pass/fail status. Exit code 4 on assertion failure. | `referential_integrity` | All values exist in a reference table column | | `custom_sql` | User-provided SQL returning violation rows | -Implementation: each rule translates to a DuckDB query against the temp_results table. Runs -after post-sql but before output. `custom_sql` allows arbitrary validation: +Implementation: each rule translates to a query (via Polars `SQLContext`) against the accumulated result. Runs after post-sql but before output. `custom_sql` allows arbitrary validation: ```yaml - rule: custom_sql @@ -765,14 +778,14 @@ dtoo query --glob "data/**/*.parquet" \ ``` **Supported databases:** -| Database | Connection String | DuckDB Support | -|----------|------------------|----------------| -| PostgreSQL | `postgres://...` | Native (postgres_scanner) | -| MySQL | `mysql://...` | Native (mysql_scanner) | -| SQLite | `sqlite:///path/to/db` | Native (sqlite_scanner) | -| Snowflake | `snowflake://account/db/schema?table=t` | Via ADBC driver | -| BigQuery | `bigquery://project/dataset?table=t` | Via ADBC driver | -| Redshift | `redshift://...` | Via Postgres wire protocol | +| Database | Connection String | +|----------|------------------| +| PostgreSQL | `postgres://...` | +| MySQL | `mysql://...` | +| SQLite | `sqlite:///path/to/db` | +| Snowflake | `snowflake://account/db/schema?table=t` | +| BigQuery | `bigquery://project/dataset?table=t` | +| Redshift | `redshift://...` (Postgres wire protocol) | **Sink modes:** - `append`: INSERT INTO target table. Schema must be compatible. @@ -802,7 +815,7 @@ output/region=GB/year=2024/part-0.parquet ... ``` -DuckDB's `COPY ... PARTITION_BY` handles this natively for Parquet and CSV. +Output is split by writing each partition subset to its own path. Partition columns are removed from the data by default (they're encoded in the path). Use `--partition-keep-columns` to retain them in the data as well. @@ -1071,9 +1084,7 @@ Process files across multiple threads for large file sets: dtoo query --glob "data/**/*.parquet" --parallel 8 --where "amount > 100" ``` -**Implementation:** Spawn N worker threads, each with its own DuckDB connection. Each worker -processes files from a shared queue and inserts into a thread-local temp table. After all files -are processed, merge thread-local tables into the final temp_results. +**Implementation:** Spawn N worker threads, each scanning files from a shared queue into thread-local `LazyFrame`s. After all files are processed, merge per-worker frames into the final accumulated result via `concat_lf_diagonal`. **Considerations:** - Default: sequential (1 thread). Enterprise unlocks `--parallel N`. @@ -1101,7 +1112,7 @@ are processed, merge thread-local tables into the final temp_results. - **Watch mode**: Re-run on file changes - **Plugin system**: Custom format readers -- **SQLite input**: Read from SQLite files via sqlite_scanner extension +- **SQLite input**: Read from SQLite files (would require a pure-Rust SQLite reader crate) - **Avro input**: Via Rust avro crate as preprocessor - **Web UI**: Browser-based dashboard for audit logs, pipeline status, and profiling reports - **dtoo server**: Long-running daemon mode for API-driven pipeline execution diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 2691b8d..f53e5f0 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4,14 +4,14 @@ This guide covers day-to-day usage of `dtoo`, from first run through practical w ## What is dtoo? -`dtoo` is a Rust CLI for querying and profiling data files with DuckDB. It is designed for fast local analytics and reproducible pipelines across file trees. +`dtoo` is a Rust CLI for querying and profiling data files. It is built on **Polars** (pure Rust) and is designed for fast local analytics and reproducible pipelines across file trees. Core capabilities: - Query many files using SQL - Read CSV, Parquet, NDJSON, and Excel (`.xlsx`, `.xls`) - Join reference tables into your query - Add lineage, masking, profiling, fingerprinting, and manifests -- Read from local files or cloud paths (S3/GCS/Azure) +- Read from local files (cloud paths S3/GCS/Azure are deferred — they return a clear error in this build) ## Install and Build @@ -94,6 +94,12 @@ dtoo query --glob "data/**/*.parquet" \ --post-sql "SELECT passenger_count, COUNT(*) AS trips FROM _ GROUP BY 1" ``` +**SQL limitations (Polars `SQLContext`):** `SELECT`, `WHERE`, `GROUP BY`, `JOIN`, `ORDER BY`, `LIMIT`, CTEs, `UNION`/`UNION ALL`, subqueries, and common string/date functions are supported. Known gaps: + +- **Window functions** (`OVER (PARTITION BY … ORDER BY …)`) have correctness issues — avoid them. +- **`DELETE`/`UPDATE`** are treated as row-filtering transforms, not DML mutations. Do not rely on DML semantics in `--post-sql`. +- Some exotic DuckDB date/regex/string functions are absent. They return a clear error (never a silent hang). + ### Excel Sheet Selection You can control sheet selection in two ways: @@ -223,21 +229,13 @@ dtoo fingerprint data/trips.parquet ## Cloud Paths -`dtoo` supports cloud URIs when credentials are configured for DuckDB extensions. - -Common options: -- `--s3-region` -- `--s3-profile` -- `--gcs-project` -- `--azure-account` - -Examples: +**Cloud storage is deferred in this build.** Paths beginning with `s3://`, `gs://`, or `az://` return an explicit error: -```bash -dtoo query --glob "s3://my-bucket/data/**/*.parquet" --s3-region us-east-1 - -dtoo query gs://my-bucket/input.csv --gcs-project my-project ``` +cloud storage (s3://…) is not supported in this build yet +``` + +The cloud CLI flags (`--s3-region`, `--s3-profile`, `--gcs-project`, `--azure-account`) still parse so that config files written for a future cloud-enabled build remain valid. ## Practical Recipes diff --git a/docs/specs/03-duckdb-engine.md b/docs/specs/03-duckdb-engine.md index ef3ebe0..772f8e6 100644 --- a/docs/specs/03-duckdb-engine.md +++ b/docs/specs/03-duckdb-engine.md @@ -1,5 +1,7 @@ # DuckDB Engine +> **Superseded by [docs/specs/34-polars-engine.md](34-polars-engine.md) — dtoo migrated from DuckDB to Polars. This spec is retained for historical context.** + > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this spec. **Goal:** Initialise and manage the in-memory DuckDB instance that powers all SQL execution, file reading, and data accumulation. diff --git a/src/crypto.rs b/src/crypto.rs index 674c1d6..153b19b 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -11,11 +11,7 @@ use polars::prelude::*; use serde::{Deserialize, Serialize}; use sha1::{Digest, Sha1}; -use crate::{ - engine::DuckDbEngine, - error::DtooError, - sql_utils::{escape_sql_literal, quote_identifier}, -}; +use crate::error::DtooError; #[derive(Debug, Clone, Deserialize)] pub struct CryptoProfile { @@ -179,135 +175,6 @@ pub fn resolve_profile( }) } -pub fn discover_wrapped_values( - engine: &DuckDbEngine, - detection: &DetectionConfig, - scoped_columns: &[String], -) -> Result, DtooError> { - let columns = detection_columns(engine, detection.mode, scoped_columns)?; - let mut rows = Vec::with_capacity(columns.len()); - - for column in columns { - let values = select_non_null_column_values(engine, &column)?; - let total = values.len(); - let encrypted = values - .iter() - .filter(|value| detect_wrapped_value(value, detection)) - .count(); - - rows.push(CryptoDiscoverRow { - column, - total, - encrypted, - }); - } - - Ok(rows) -} - -pub fn decrypt_temp_results( - engine: &DuckDbEngine, - profile: &CryptoProfile, -) -> Result { - validate_detection_config(&profile.detection)?; - - if profile.scheme.scheme_type == SchemeType::LegacyAes128EcbSha1 { - eprintln!( - "Warning: legacy_aes128_ecb_sha1 uses AES-ECB with no integrity protection. Use aes256_gcm_base64 for new data." - ); - } - - let columns = detection_columns(engine, profile.detection.mode, &profile.columns)?; - let scheme = scheme_impl(profile.scheme.scheme_type); - let key_material = load_key_material(profile)?; - - let mut decrypted_columns = Vec::new(); - - for column in columns { - let values = select_non_null_column_values(engine, &column)?; - let mut changed = false; - - for original in values { - if !detect_wrapped_value(&original, &profile.detection) { - continue; - } - let Some(inner) = strip_wrapper(&original, &profile.detection) else { - continue; - }; - if !scheme.detect_inner(inner) { - continue; - } - - match scheme.decrypt_inner(inner, &key_material) { - Ok(plaintext) => { - if plaintext != original { - update_column_value(engine, &column, &original, &plaintext)?; - changed = true; - } - } - Err(err) => { - if profile.failure_mode == FailureMode::Strict { - return Err(DtooError::Config { - message: format!( - "decryption failed for column `{column}` using profile `{}`: {err}", - profile.name - ), - }); - } - eprintln!( - "Warning: decryption failed in column `{column}` using profile `{}`: {err}", - profile.name - ); - } - } - } - - if changed { - decrypted_columns.push(column); - } - } - - Ok(CryptoProcessResult { decrypted_columns }) -} - -pub fn encrypt_columns( - engine: &DuckDbEngine, - profile: &CryptoProfile, - columns: &[String], -) -> Result<(), DtooError> { - if columns.is_empty() { - return Ok(()); - } - - validate_detection_config(&profile.detection)?; - let available = string_columns(engine)?; - let available_set = available.iter().cloned().collect::>(); - for column in columns { - if !available_set.contains(column) { - return Err(DtooError::Config { - message: format!("encrypt column `{column}` not found or not string-like"), - }); - } - } - - let scheme = scheme_impl(profile.scheme.scheme_type); - let key_material = load_key_material(profile)?; - - for column in columns { - let values = select_non_null_column_values(engine, column)?; - for plaintext in values { - if detect_wrapped_value(&plaintext, &profile.detection) { - continue; - } - let inner = scheme.encrypt_inner(&plaintext, &key_material)?; - let wrapped = wrap_inner(&inner, &profile.detection); - update_column_value(engine, column, &plaintext, &wrapped)?; - } - } - - Ok(()) -} - pub fn enforce_output_safety( decrypted_columns: &[String], allow_plaintext_pii: bool, @@ -394,83 +261,6 @@ fn scheme_impl(scheme_type: SchemeType) -> Box { } } -fn detection_columns( - engine: &DuckDbEngine, - mode: DetectionMode, - scoped_columns: &[String], -) -> Result, DtooError> { - let all = string_columns(engine)?; - let all_set = all.iter().cloned().collect::>(); - - let from_scoped = scoped_columns - .iter() - .filter(|column| all_set.contains(*column)) - .cloned() - .collect::>(); - - Ok(match mode { - DetectionMode::Auto => all, - DetectionMode::Columns => from_scoped, - DetectionMode::AutoOrColumns => { - if from_scoped.is_empty() { - all - } else { - from_scoped - } - } - }) -} - -fn string_columns(engine: &DuckDbEngine) -> Result, DtooError> { - let rows = engine.query( - "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'temp_results' ORDER BY ordinal_position", - )?; - - let mut out = Vec::new(); - for row in rows { - let Some(name) = row.values.first() else { - continue; - }; - let data_type = row.values.get(1).map(String::as_str).unwrap_or_default(); - let upper = data_type.to_ascii_uppercase(); - if upper.contains("CHAR") || upper.contains("TEXT") || upper.contains("STRING") { - out.push(name.clone()); - } - } - - Ok(out) -} - -fn select_non_null_column_values( - engine: &DuckDbEngine, - column: &str, -) -> Result, DtooError> { - let sql = format!( - "SELECT DISTINCT {} FROM temp_results WHERE {} IS NOT NULL", - quote_identifier(column), - quote_identifier(column) - ); - let rows = engine.query(&sql)?; - Ok(rows - .into_iter() - .filter_map(|row| row.values.into_iter().next()) - .collect()) -} - -fn update_column_value( - engine: &DuckDbEngine, - column: &str, - from_value: &str, - to_value: &str, -) -> Result<(), DtooError> { - let qcol = quote_identifier(column); - let from_escaped = escape_sql_literal(from_value); - let to_escaped = escape_sql_literal(to_value); - let sql = - format!("UPDATE temp_results SET {qcol} = '{to_escaped}' WHERE {qcol} = '{from_escaped}'"); - engine.execute(&sql) -} - fn detect_wrapped_value(value: &str, detection: &DetectionConfig) -> bool { let candidate = if detection.trim_whitespace { value.trim() @@ -701,8 +491,7 @@ fn decode_32b_key_base64(key_material: &str) -> Result<[u8; 32], DtooError> { // --------------------------------------------------------------------------- /// Returns the names of all `DataType::String` columns in `df`. -/// Mirrors what `string_columns(engine)` does for the DuckDB `temp_results` -/// table: only VARCHAR/TEXT/STRING columns are eligible for crypto operations. +/// Only String columns are eligible for crypto operations. fn string_columns_df(df: &DataFrame) -> Vec { df.get_column_names() .into_iter() @@ -711,8 +500,8 @@ fn string_columns_df(df: &DataFrame) -> Vec { .collect() } -/// Decides which columns to operate on for a DataFrame, mirroring -/// `detection_columns(engine, mode, scoped_columns)`. +/// Decides which columns to operate on for a DataFrame based on the detection +/// mode and any scoped columns. fn detection_columns_df( df: &DataFrame, mode: DetectionMode, @@ -740,8 +529,7 @@ fn detection_columns_df( } } -/// Collect all non-null distinct string values for a column as owned `String`s. -/// Mirrors `select_non_null_column_values(engine, col)`. +/// Collect all non-null string values for a column as owned `String`s. fn column_non_null_values_df(df: &DataFrame, column: &str) -> Result, DtooError> { let series = df.column(column).map_err(|e| DtooError::Config { message: format!("column `{column}` not found in DataFrame: {e}"), @@ -759,7 +547,7 @@ fn column_non_null_values_df(df: &DataFrame, column: &str) -> Result } /// Replace every occurrence of `from_value` with `to_value` in the named column, -/// mutating `df` in place. Mirrors `update_column_value(engine, col, from, to)`. +/// mutating `df` in place. fn replace_column_value_df( df: &mut DataFrame, column: &str, @@ -786,11 +574,8 @@ fn replace_column_value_df( }) } -/// DataFrame-based equivalent of `discover_wrapped_values`. -/// /// Scans the String-typed columns of `df` (filtered by `detection.mode` and -/// `scoped_columns`) and returns per-column discovery counts. The per-value -/// detection logic (`detect_wrapped_value`) is reused unchanged. +/// `scoped_columns`) and returns per-column discovery counts. pub fn discover_wrapped_values_df( df: &DataFrame, detection: &DetectionConfig, @@ -816,13 +601,9 @@ pub fn discover_wrapped_values_df( Ok(rows) } -/// DataFrame-based equivalent of `decrypt_temp_results`. -/// /// Decrypts every detected-wrapped value in the eligible String columns of `df` /// and returns the updated DataFrame together with a `CryptoProcessResult` -/// listing which columns were modified. The per-value decrypt logic -/// (`CryptoScheme::decrypt_inner`, `detect_wrapped_value`, `strip_wrapper`, -/// `FailureMode`) is reused unchanged. +/// listing which columns were modified. pub fn decrypt_dataframe( mut df: DataFrame, profile: &CryptoProfile, @@ -888,12 +669,8 @@ pub fn decrypt_dataframe( Ok((df, CryptoProcessResult { decrypted_columns })) } -/// DataFrame-based equivalent of `encrypt_columns`. -/// /// Encrypts every non-wrapped plaintext value in the specified `columns` of -/// `df` and returns the updated DataFrame. The per-value encrypt logic -/// (`CryptoScheme::encrypt_inner`, `wrap_inner`, `detect_wrapped_value`) is -/// reused unchanged. +/// `df` and returns the updated DataFrame. pub fn encrypt_dataframe( mut df: DataFrame, profile: &CryptoProfile, diff --git a/src/engine.rs b/src/engine.rs deleted file mode 100644 index cab0576..0000000 --- a/src/engine.rs +++ /dev/null @@ -1,772 +0,0 @@ -use std::collections::HashSet; - -use duckdb::{Connection, arrow::record_batch::RecordBatch, types::Value}; - -use crate::{ - error::DtooError, - sql_utils::{escape_sql_literal, quote_identifier}, - types::{CompressionCodec, ExportFormat, InputFormat, SchemaColumn}, -}; - -/// Optional cloud settings applied during engine initialisation. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct CloudSettings { - pub s3_region: Option, - pub s3_profile: Option, - pub s3_access_key_id: Option, - pub gcs_project_id: Option, - pub azure_storage_account_name: Option, -} - -/// Engine initialisation options. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct EngineConfig { - pub cloud: CloudSettings, - pub load_cloud_extensions: bool, - pub load_excel_extension: bool, -} - -/// A materialized row representation for generic query access. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct QueryRow { - pub values: Vec, -} - -/// A batch wrapper for bulk query output. -#[derive(Clone, Debug)] -pub struct ArrowBatch { - pub batches: Vec, -} - -impl ArrowBatch { - /// Returns the number of rows in this batch. - pub fn row_count(&self) -> usize { - self.batches.iter().map(RecordBatch::num_rows).sum() - } -} - -/// Thin wrapper around an in-memory DuckDB connection. -pub struct DuckDbEngine { - connection: Connection, -} - -impl DuckDbEngine { - /// Initialise a new in-memory DuckDB engine. - pub fn new(config: EngineConfig) -> Result { - let connection = Connection::open_in_memory().map_err(|source| DtooError::Output { - message: format!("failed to open in-memory DuckDB: {source}"), - })?; - - let engine = Self { connection }; - - if config.load_cloud_extensions { - engine.install_and_load_extension("httpfs")?; - engine.install_and_load_extension("azure")?; - } - if config.load_excel_extension { - engine.install_and_load_extension("excel")?; - } - - engine.apply_cloud_settings(&config.cloud)?; - - Ok(engine) - } - - /// Execute a SQL statement that does not return result rows. - pub fn execute(&self, sql: &str) -> Result<(), DtooError> { - self.connection - .execute_batch(sql) - .map_err(|source| DtooError::Sql { - context: "statement".to_string(), - sql: sql.to_string(), - source: Box::new(source), - }) - } - - /// Execute DML and return affected row count. - pub fn execute_with_count(&self, sql: &str) -> Result { - self.connection - .execute(sql, []) - .map_err(|source| DtooError::Sql { - context: "statement".to_string(), - sql: sql.to_string(), - source: Box::new(source), - }) - } - - /// Execute a SELECT query and return stringified row values. - pub fn query(&self, sql: &str) -> Result, DtooError> { - let mut stmt = self - .connection - .prepare(sql) - .map_err(|source| DtooError::Sql { - context: "query".to_string(), - sql: sql.to_string(), - source: Box::new(source), - })?; - - let mapped_rows = stmt - .query_map([], |row| { - let column_count = row.as_ref().column_count(); - let mut values = Vec::with_capacity(column_count); - for idx in 0..column_count { - let value: Value = row.get(idx)?; - values.push(value_to_string(value)); - } - Ok(QueryRow { values }) - }) - .map_err(|source| DtooError::Sql { - context: "query".to_string(), - sql: sql.to_string(), - source: Box::new(source), - })?; - - let mut out = Vec::new(); - for row in mapped_rows { - let row = row.map_err(|source| DtooError::Sql { - context: "query".to_string(), - sql: sql.to_string(), - source: Box::new(source), - })?; - out.push(row); - } - - Ok(out) - } - - /// Execute a query and return a batch container for bulk movement. - pub fn query_arrow(&self, sql: &str) -> Result { - let mut stmt = self - .connection - .prepare(sql) - .map_err(|source| DtooError::Sql { - context: "query".to_string(), - sql: sql.to_string(), - source: Box::new(source), - })?; - - let batches = stmt - .query_arrow([]) - .map_err(|source| DtooError::Sql { - context: "query".to_string(), - sql: sql.to_string(), - source: Box::new(source), - })? - .collect::>(); - - Ok(ArrowBatch { batches }) - } - - /// Execute a query that returns a single count-like numeric value. - pub fn query_count(&self, sql: &str) -> Result { - let rows = self.query(sql)?; - let value = rows - .first() - .and_then(|row| row.values.first()) - .ok_or_else(|| DtooError::Sql { - context: "query".to_string(), - sql: sql.to_string(), - source: Box::new(std::io::Error::other("no count row returned")), - })?; - - value.parse::().map_err(|source| DtooError::Sql { - context: "query".to_string(), - sql: sql.to_string(), - source: Box::new(source), - }) - } - - /// Register an input file as the magic `_` view. - pub fn register_magic_table(&self, path: &str, format: &InputFormat) -> Result<(), DtooError> { - let escaped_path = escape_sql_literal(path); - let sql = match format { - InputFormat::Parquet => { - format!("CREATE OR REPLACE VIEW _ AS SELECT * FROM read_parquet('{escaped_path}')") - } - InputFormat::Csv { delimiter } => { - let delim = escape_sql_literal(&delimiter.to_string()); - format!( - "CREATE OR REPLACE VIEW _ AS SELECT * FROM read_csv('{escaped_path}', delim='{delim}', header=true, auto_detect=true)" - ) - } - InputFormat::Ndjson => { - format!( - "CREATE OR REPLACE VIEW _ AS SELECT * FROM read_ndjson_auto('{escaped_path}')" - ) - } - InputFormat::Excel { sheet } => { - if let Some(sheet) = sheet { - let escaped_sheet = escape_sql_literal(sheet); - format!( - "CREATE OR REPLACE VIEW _ AS SELECT * FROM read_xlsx('{escaped_path}', sheet='{escaped_sheet}')" - ) - } else { - format!("CREATE OR REPLACE VIEW _ AS SELECT * FROM read_xlsx('{escaped_path}')") - } - } - }; - - self.execute(&sql) - } - - /// Load a reference file into a named DuckDB table and return inserted row count. - pub fn create_reference_table( - &self, - table_name: &str, - path: &str, - format: &InputFormat, - ) -> Result { - let escaped_path = escape_sql_literal(path); - let quoted_name = quote_identifier(table_name); - let sql = match format { - InputFormat::Parquet => format!( - "CREATE TABLE {quoted_name} AS SELECT * FROM read_parquet('{escaped_path}')" - ), - InputFormat::Csv { delimiter } => { - let delim = escape_sql_literal(&delimiter.to_string()); - format!( - "CREATE TABLE {quoted_name} AS SELECT * FROM read_csv('{escaped_path}', delim='{delim}', header=true, auto_detect=true)" - ) - } - InputFormat::Ndjson => format!( - "CREATE TABLE {quoted_name} AS SELECT * FROM read_ndjson_auto('{escaped_path}')" - ), - InputFormat::Excel { sheet } => { - if let Some(sheet) = sheet { - let escaped_sheet = escape_sql_literal(sheet); - format!( - "CREATE TABLE {quoted_name} AS SELECT * FROM read_xlsx('{escaped_path}', sheet='{escaped_sheet}')" - ) - } else { - format!( - "CREATE TABLE {quoted_name} AS SELECT * FROM read_xlsx('{escaped_path}')" - ) - } - } - }; - - self.execute(&sql)?; - self.query_count(&format!("SELECT COUNT(*) FROM {quoted_name}")) - } - - /// Create `temp_results` from a query shape and insert the current rows by name. - pub fn create_temp_results_from_query(&self, query_sql: &str) -> Result<(), DtooError> { - self.create_temp_results_schema_from_query(query_sql)?; - self.insert_into_temp_results_by_name(query_sql)?; - Ok(()) - } - - /// Create `temp_results` schema from the shape of a query. - pub fn create_temp_results_schema_from_query(&self, query_sql: &str) -> Result<(), DtooError> { - let create_sql = - format!("CREATE TABLE temp_results AS SELECT * FROM ({query_sql}) q LIMIT 0"); - self.execute(&create_sql) - } - - /// Insert rows into `temp_results` using DuckDB's `BY NAME` mapping. - pub fn insert_into_temp_results_by_name(&self, query_sql: &str) -> Result { - let insert_sql = format!("INSERT INTO temp_results BY NAME SELECT * FROM ({query_sql}) q"); - self.execute_with_count(&insert_sql) - } - - /// Merge rows into `temp_results` using `UNION ALL BY NAME` to evolve schema safely. - pub fn merge_into_temp_results_by_name(&self, query_sql: &str) -> Result { - let source_count = self.query_count(&format!("SELECT COUNT(*) FROM ({query_sql}) q"))?; - if !self.query_introduces_new_temp_results_columns(query_sql)? { - self.insert_into_temp_results_by_name(query_sql)?; - return Ok(source_count); - } - - let merge_sql = format!( - "CREATE TABLE temp_results_next AS \ - SELECT * FROM temp_results \ - UNION ALL BY NAME \ - SELECT * FROM ({query_sql}) q" - ); - self.execute(&merge_sql)?; - self.execute("DROP TABLE temp_results")?; - self.execute("ALTER TABLE temp_results_next RENAME TO temp_results")?; - Ok(source_count) - } - - fn query_introduces_new_temp_results_columns( - &self, - query_sql: &str, - ) -> Result { - let existing = self.temp_results_columns()?; - let incoming = self.query_columns(query_sql)?; - Ok(incoming.into_iter().any(|name| !existing.contains(&name))) - } - - fn temp_results_columns(&self) -> Result, DtooError> { - let rows = self.query("DESCRIBE temp_results")?; - Ok(rows - .iter() - .filter_map(|row| row.values.first()) - .map(|name| name.to_ascii_lowercase()) - .collect()) - } - - fn query_columns(&self, query_sql: &str) -> Result, DtooError> { - let rows = self.query(&format!("DESCRIBE SELECT * FROM ({query_sql}) q"))?; - Ok(rows - .iter() - .filter_map(|row| row.values.first()) - .map(|name| name.to_ascii_lowercase()) - .collect()) - } - - /// Create `temp_results` from an explicit schema definition. - pub fn create_temp_results_from_schema( - &self, - columns: &[SchemaColumn], - ) -> Result<(), DtooError> { - if columns.is_empty() { - return Err(DtooError::Schema { - message: "explicit schema must contain at least one column".to_string(), - }); - } - - let definition = columns - .iter() - .map(|column| format!("{} {}", quote_identifier(&column.name), column.data_type)) - .collect::>() - .join(", "); - - let sql = format!("CREATE TABLE temp_results ({definition})"); - self.execute(&sql) - } - - /// Export `temp_results` to path or stdout. - pub fn export_results( - &self, - destination: Option<&str>, - format: ExportFormat, - header: bool, - delimiter: char, - compression: Option, - ) -> Result<(), DtooError> { - let target = destination.unwrap_or("/dev/stdout"); - let escaped_target = escape_sql_literal(target); - - let options = match format { - ExportFormat::Csv => { - let delim = escape_sql_literal(&delimiter.to_string()); - let mut parts = vec![ - "FORMAT CSV".to_string(), - format!("HEADER {}", if header { "true" } else { "false" }), - format!("DELIMITER '{delim}'"), - ]; - if let Some(codec) = compression { - parts.push(format!("COMPRESSION '{}'", compression_to_sql(codec))); - } - parts.join(", ") - } - ExportFormat::Parquet => { - let mut parts = vec!["FORMAT PARQUET".to_string()]; - if let Some(codec) = compression { - parts.push(format!("COMPRESSION '{}'", compression_to_sql(codec))); - } - parts.join(", ") - } - ExportFormat::Ndjson => { - let mut parts = vec!["FORMAT JSON".to_string()]; - if let Some(codec) = compression { - parts.push(format!("COMPRESSION '{}'", compression_to_sql(codec))); - } - parts.join(", ") - } - }; - - let sql = format!("COPY (SELECT * FROM temp_results) TO '{escaped_target}' ({options})"); - self.execute(&sql) - } - - fn install_and_load_extension(&self, extension: &str) -> Result<(), DtooError> { - if !is_safe_extension_name(extension) { - return Err(DtooError::Config { - message: format!("invalid extension name: {extension}"), - }); - } - - let install_sql = format!("INSTALL {extension}"); - let install_result = self.connection.execute_batch(&install_sql); - - let load_sql = format!("LOAD {extension}"); - if let Err(load_error) = self.connection.execute_batch(&load_sql) { - let source = if let Err(install_error) = install_result { - std::io::Error::other(format!( - "INSTALL failed: {install_error}; LOAD failed: {load_error}" - )) - } else { - std::io::Error::other(load_error.to_string()) - }; - - return Err(DtooError::ExtensionLoad { - extension: extension.to_string(), - source: Box::new(source), - }); - } - - if let Err(install_error) = install_result { - eprintln!( - "Warning: INSTALL {extension} failed, but LOAD succeeded (likely already cached): {install_error}" - ); - } - - Ok(()) - } - - fn apply_cloud_settings(&self, cloud: &CloudSettings) -> Result<(), DtooError> { - if let Some(region) = &cloud.s3_region { - self.set_option("s3_region", region)?; - } - if let Some(profile) = &cloud.s3_profile { - self.set_option("s3_profile", profile)?; - } - if let Some(access_key_id) = &cloud.s3_access_key_id { - self.set_option("s3_access_key_id", access_key_id)?; - } - if let Some(project) = &cloud.gcs_project_id { - self.set_option("gcs_project_id", project)?; - } - if let Some(account_name) = &cloud.azure_storage_account_name { - self.set_option("azure_storage_account_name", account_name)?; - } - - Ok(()) - } - - fn set_option(&self, key: &str, value: &str) -> Result<(), DtooError> { - let escaped = escape_sql_literal(value); - let sql = format!("SET {key} = '{escaped}'"); - self.execute(&sql).map_err(|source| match source { - DtooError::Sql { .. } => DtooError::Config { - message: format!("failed to configure {key}"), - }, - other => other, - }) - } -} - -fn compression_to_sql(codec: CompressionCodec) -> &'static str { - match codec { - CompressionCodec::Gzip => "gzip", - CompressionCodec::Zstd => "zstd", - } -} - -fn value_to_string(value: Value) -> String { - match value { - Value::Null => "NULL".to_string(), - Value::Boolean(v) => v.to_string(), - Value::TinyInt(v) => v.to_string(), - Value::SmallInt(v) => v.to_string(), - Value::Int(v) => v.to_string(), - Value::BigInt(v) => v.to_string(), - Value::HugeInt(v) => v.to_string(), - Value::UTinyInt(v) => v.to_string(), - Value::USmallInt(v) => v.to_string(), - Value::UInt(v) => v.to_string(), - Value::UBigInt(v) => v.to_string(), - Value::Float(v) => v.to_string(), - Value::Double(v) => v.to_string(), - Value::Decimal(v) => v.to_string(), - Value::Timestamp(unit, v) => format!("{v} ({unit:?})"), - Value::Text(v) => v, - Value::Blob(v) => format!("blob({} bytes)", v.len()), - Value::Date32(v) => v.to_string(), - Value::Time64(_, v) => v.to_string(), - Value::Interval { - months, - days, - nanos, - } => format!("{months} months {days} days {nanos} nanos"), - Value::List(v) => format!("{v:?}"), - Value::Enum(v) => v, - Value::Struct(v) => format!("{v:?}"), - Value::Array(v) => format!("{v:?}"), - Value::Map(v) => format!("{v:?}"), - Value::Union(v) => format!("{v:?}"), - } -} - -fn is_safe_extension_name(input: &str) -> bool { - !input.is_empty() && input.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') -} - -#[cfg(test)] -mod tests { - use super::*; - - fn test_engine() -> DuckDbEngine { - DuckDbEngine::new(EngineConfig { - cloud: CloudSettings::default(), - load_cloud_extensions: false, - load_excel_extension: false, - }) - .expect("engine init should succeed") - } - - #[test] - fn execute_and_query_roundtrip() { - let engine = test_engine(); - engine - .execute("CREATE TABLE t (id INTEGER, name VARCHAR)") - .expect("create table should succeed"); - let inserted = engine - .execute_with_count("INSERT INTO t VALUES (1, 'alice'), (2, 'bob')") - .expect("insert should succeed"); - assert_eq!(inserted, 2); - - let rows = engine - .query("SELECT id, name FROM t ORDER BY id") - .expect("query should succeed"); - - assert_eq!(rows.len(), 2); - assert_eq!(rows[0].values, vec!["1", "alice"]); - assert_eq!(rows[1].values, vec!["2", "bob"]); - } - - #[test] - fn query_arrow_returns_rows() { - let engine = test_engine(); - engine - .execute("CREATE TABLE t (id INTEGER)") - .expect("create table should succeed"); - engine - .execute("INSERT INTO t VALUES (1), (2), (3)") - .expect("insert should succeed"); - - let batch = engine - .query_arrow("SELECT id FROM t ORDER BY id") - .expect("query arrow should succeed"); - - assert_eq!(batch.row_count(), 3); - } - - #[test] - fn create_temp_results_from_explicit_schema() { - let engine = test_engine(); - engine - .create_temp_results_from_schema(&[ - SchemaColumn { - name: "id".to_string(), - data_type: "INTEGER".to_string(), - }, - SchemaColumn { - name: "name".to_string(), - data_type: "VARCHAR".to_string(), - }, - ]) - .expect("schema creation should succeed"); - - let rows = engine - .query("PRAGMA table_info('temp_results')") - .expect("table info query should succeed"); - assert_eq!(rows.len(), 2); - } - - #[test] - fn create_temp_results_from_query_uses_by_name_insert() { - let engine = test_engine(); - engine - .execute("CREATE VIEW _ AS SELECT 1 AS id, 'alice' AS name") - .expect("create view should succeed"); - - engine - .create_temp_results_from_query("SELECT id, name FROM _") - .expect("temp results should be created"); - - let rows = engine - .query("SELECT id, name FROM temp_results") - .expect("query temp results should succeed"); - - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].values, vec!["1", "alice"]); - } - - #[test] - fn merge_uses_insert_fast_path_when_no_new_columns() { - let engine = test_engine(); - engine - .execute("CREATE VIEW _ AS SELECT 1 AS id, 'alice' AS name") - .expect("create view should succeed"); - engine - .create_temp_results_from_query("SELECT id, name FROM _") - .expect("create temp results"); - - let inserted = engine - .merge_into_temp_results_by_name("SELECT 2 AS id, 'bob' AS name") - .expect("merge should succeed"); - assert_eq!(inserted, 1); - - let rows = engine - .query("SELECT id, name FROM temp_results ORDER BY id") - .expect("query should succeed"); - assert_eq!(rows.len(), 2); - assert_eq!(rows[0].values, vec!["1", "alice"]); - assert_eq!(rows[1].values, vec!["2", "bob"]); - } - - #[test] - fn merge_evolves_schema_when_new_columns_arrive() { - let engine = test_engine(); - engine - .execute("CREATE VIEW _ AS SELECT 1 AS id") - .expect("create view should succeed"); - engine - .create_temp_results_from_query("SELECT id FROM _") - .expect("create temp results"); - - engine - .merge_into_temp_results_by_name("SELECT 2 AS id, 'x' AS extra") - .expect("merge should evolve schema"); - - let rows = engine - .query("DESCRIBE temp_results") - .expect("describe should succeed"); - let names = rows - .iter() - .filter_map(|row| row.values.first()) - .cloned() - .collect::>(); - assert!(names.contains(&"id".to_string())); - assert!(names.contains(&"extra".to_string())); - } - - #[test] - fn register_magic_table_supports_inline_csv() { - let engine = test_engine(); - let csv_path = std::env::temp_dir().join(format!( - "dtoo-magic-{}.csv", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("clock should be after epoch") - .as_nanos() - )); - std::fs::write(&csv_path, "id,name\n1,alice\n").expect("temp csv should be written"); - - engine - .register_magic_table( - csv_path.to_string_lossy().as_ref(), - &InputFormat::Csv { delimiter: ',' }, - ) - .expect("registering csv should succeed"); - - let rows = engine - .query("SELECT id, name FROM _") - .expect("query _ should work"); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].values, vec!["1", "alice"]); - let _ = std::fs::remove_file(csv_path); - } - - #[test] - fn create_reference_table_loads_csv() { - let engine = test_engine(); - let csv_path = std::env::temp_dir().join(format!( - "dtoo-ref-{}.csv", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("clock should be after epoch") - .as_nanos() - )); - std::fs::write(&csv_path, "id,name\n1,alice\n").expect("temp csv should be written"); - - let row_count = engine - .create_reference_table( - "regions", - csv_path.to_string_lossy().as_ref(), - &InputFormat::Csv { delimiter: ',' }, - ) - .expect("reference table load should succeed"); - assert_eq!(row_count, 1); - - let rows = engine - .query("SELECT id, name FROM regions") - .expect("query regions should succeed"); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].values, vec!["1", "alice"]); - let _ = std::fs::remove_file(csv_path); - } - - #[test] - fn export_results_writes_csv_file() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (id INTEGER, name VARCHAR)") - .expect("create table should succeed"); - engine - .execute("INSERT INTO temp_results VALUES (1, 'alice')") - .expect("insert should succeed"); - - let path = std::env::temp_dir().join(format!( - "dtoo-export-{}.csv", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("clock should be after epoch") - .as_nanos() - )); - - engine - .export_results( - Some(path.to_string_lossy().as_ref()), - ExportFormat::Csv, - true, - ',', - None, - ) - .expect("export should succeed"); - - let contents = std::fs::read_to_string(&path).expect("exported file should be readable"); - assert!(contents.contains("id,name")); - assert!(contents.contains("1,alice")); - - let _ = std::fs::remove_file(path); - } - - #[test] - fn query_returns_sql_error_for_invalid_statement() { - let engine = test_engine(); - let err = engine - .query("SELECT definitely_not_a_column FROM missing_table") - .expect_err("query should fail"); - - assert!(matches!(err, DtooError::Sql { .. })); - } - - #[test] - fn register_magic_table_returns_file_context_for_missing_file() { - let engine = test_engine(); - let err = engine - .register_magic_table( - "/tmp/does-not-exist-123.csv", - &InputFormat::Csv { delimiter: ',' }, - ) - .expect_err("registration should fail"); - - match err { - DtooError::Sql { sql, .. } => { - assert!(sql.contains("read_csv")); - assert!(sql.contains("/tmp/does-not-exist-123.csv")); - } - other => panic!("expected SQL error, got {other:?}"), - } - } - - #[test] - fn rejects_unsafe_extension_names() { - let engine = test_engine(); - let err = engine - .install_and_load_extension("httpfs; DROP TABLE temp_results;") - .expect_err("unsafe extension name must be rejected"); - - match err { - DtooError::Config { message } => { - assert!(message.contains("invalid extension name")); - } - other => panic!("expected config error, got {other:?}"), - } - } -} diff --git a/src/file_resolution.rs b/src/file_resolution.rs index c700069..a2a2722 100644 --- a/src/file_resolution.rs +++ b/src/file_resolution.rs @@ -5,14 +5,12 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; -use duckdb::Connection; use glob::Pattern; use crate::{ cli::{OnErrorMode, PipeMode, StdinFormat}, error::DtooError, path_utils::{is_cloud_path, split_excel_sheet_from_path}, - sql_utils::escape_sql_literal, }; /// A resolved input file with normalized metadata. @@ -172,59 +170,9 @@ impl FileResolver { } fn resolve_cloud_glob(&self, pattern: &str) -> Result, DtooError> { - let conn = Connection::open_in_memory().map_err(|source| DtooError::Output { - message: format!("failed to initialize cloud glob resolver: {source}"), - })?; - - let _ = conn.execute_batch("INSTALL httpfs; LOAD httpfs;"); - let _ = conn.execute_batch("INSTALL azure; LOAD azure;"); - - if let Some(region) = &self.config.s3_region { - set_sql_option(&conn, "s3_region", region)?; - } - if let Some(profile) = &self.config.s3_profile { - set_sql_option(&conn, "s3_profile", profile)?; - } - if let Some(project) = &self.config.gcs_project { - set_sql_option(&conn, "gcs_project_id", project)?; - } - if let Some(account) = &self.config.azure_account { - set_sql_option(&conn, "azure_storage_account_name", account)?; - } - - let sql = format!("SELECT * FROM glob('{}')", escape_sql_literal(pattern)); - let mut stmt = conn.prepare(&sql).map_err(|source| DtooError::Sql { - context: "glob".to_string(), - sql: sql.clone(), - source: Box::new(source), - })?; - let mut rows = stmt.query([]).map_err(|source| DtooError::Sql { - context: "glob".to_string(), - sql: sql.clone(), - source: Box::new(source), - })?; - - let mut files = Vec::new(); - while let Some(row) = rows.next().map_err(|source| DtooError::Sql { - context: "glob".to_string(), - sql: sql.clone(), - source: Box::new(source), - })? { - let path: String = row.get(0).map_err(|source| DtooError::Sql { - context: "glob".to_string(), - sql: sql.clone(), - source: Box::new(source), - })?; - if !is_supported_data_path(&path) { - continue; - } - files.push(CandidatePath { - path, - is_temp: false, - }); - } - - Ok(files) + Err(DtooError::Config { + message: format!("cloud storage glob ({pattern}) is not supported in this build yet"), + }) } fn resolve_from_pipe_file( @@ -385,15 +333,6 @@ impl FileResolver { } } -fn set_sql_option(connection: &Connection, key: &str, value: &str) -> Result<(), DtooError> { - let sql = format!("SET {key} = '{}'", escape_sql_literal(value)); - connection - .execute_batch(&sql) - .map_err(|source| DtooError::Config { - message: format!("failed to configure {key}: {source}"), - }) -} - fn is_supported_data_path(path: &str) -> bool { let lower = path.to_ascii_lowercase(); lower.ends_with(".parquet") @@ -658,6 +597,25 @@ mod tests { assert!(matches!(err, DtooError::FileNotFound { .. })); } + #[test] + fn cloud_glob_is_rejected_with_clear_error() { + let config = FileResolverConfig { + glob: Some("s3://bucket/*.parquet".to_string()), + ..FileResolverConfig::default() + }; + let resolver = FileResolver::new(config); + let err = resolver + .resolve_with_reader(&mut io::empty()) + .expect_err("cloud glob should error"); + match err { + DtooError::Config { message } => { + assert!(message.contains("cloud storage")); + assert!(message.contains("not supported")); + } + other => panic!("expected Config error, got {other:?}"), + } + } + fn temp_dir(prefix: &str) -> std::path::PathBuf { let dir = std::env::temp_dir().join(format!("{prefix}-{}", unix_nanos())); fs::create_dir_all(&dir).expect("temp dir should be created"); diff --git a/src/fingerprint.rs b/src/fingerprint.rs index d1c413a..08e1b16 100644 --- a/src/fingerprint.rs +++ b/src/fingerprint.rs @@ -1,24 +1,21 @@ -use std::{ - fs::File, - io::Read, - path::{Path, PathBuf}, -}; +use std::{fs::File, io::Read, path::Path}; -use duckdb::Connection; use sha2::{Digest, Sha256}; -use crate::{error::DtooError, path_utils::is_cloud_path_buf, sql_utils::escape_sql_literal}; +use crate::{error::DtooError, path_utils::is_cloud_path_buf}; -/// Compute `sha256:` for a local or cloud-backed path. +/// Compute `sha256:` for a local path. pub fn fingerprint_file(path: &Path) -> Result { - let target = if is_cloud_path_buf(path) { - download_cloud_blob(path)? - } else { - path.to_path_buf() - }; - + if is_cloud_path_buf(path) { + return Err(DtooError::Config { + message: format!( + "cloud storage ({}) is not supported in this build yet", + path.to_string_lossy() + ), + }); + } let display_path = path.to_string_lossy().to_string(); - let mut file = File::open(&target).map_err(|source| map_io_error(&display_path, source))?; + let mut file = File::open(path).map_err(|source| map_io_error(&display_path, source))?; let mut hasher = Sha256::new(); let mut buffer = [0u8; 8192]; loop { @@ -30,11 +27,6 @@ pub fn fingerprint_file(path: &Path) -> Result { } hasher.update(&buffer[..bytes]); } - - if is_cloud_path_buf(path) { - let _ = std::fs::remove_file(&target); - } - Ok(format!("sha256:{}", hex::encode(hasher.finalize()))) } @@ -53,51 +45,6 @@ pub fn display_name(path: &Path) -> String { .unwrap_or_else(|| path.to_string_lossy().to_string()) } -fn download_cloud_blob(path: &Path) -> Result { - let conn = Connection::open_in_memory().map_err(|source| DtooError::Output { - message: format!("failed to initialize cloud fingerprint reader: {source}"), - })?; - - let _ = conn.execute_batch("INSTALL httpfs; LOAD httpfs;"); - let _ = conn.execute_batch("INSTALL azure; LOAD azure;"); - - let sql = format!( - "SELECT content FROM read_blob('{}') LIMIT 1", - escape_sql_literal(path.to_string_lossy().as_ref()) - ); - let mut stmt = conn.prepare(&sql).map_err(|source| DtooError::Sql { - context: "fingerprint".to_string(), - sql: sql.clone(), - source: Box::new(source), - })?; - let bytes: Vec = stmt.query_row([], |row| row.get(0)).map_err(|source| { - let rendered = source.to_string(); - if should_map_missing_credentials(&rendered) { - return DtooError::Config { - message: format!("{} credentials not configured", cloud_provider(path)), - }; - } - DtooError::Sql { - context: "fingerprint".to_string(), - sql, - source: Box::new(source), - } - })?; - - let temp = std::env::temp_dir().join(format!( - "dtoo-fingerprint-{}.bin", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("clock after epoch") - .as_nanos() - )); - std::fs::write(&temp, bytes).map_err(|source| DtooError::FileRead { - path: temp.display().to_string(), - source: Box::new(source), - })?; - Ok(temp) -} - fn map_io_error(path: &str, source: std::io::Error) -> DtooError { match source.kind() { std::io::ErrorKind::NotFound => DtooError::FileNotFound { @@ -113,26 +60,6 @@ fn map_io_error(path: &str, source: std::io::Error) -> DtooError { } } -fn should_map_missing_credentials(message: &str) -> bool { - let lower = message.to_ascii_lowercase(); - lower.contains("credential") - || lower.contains("access key") - || lower.contains("secret key") - || lower.contains("no provider") - || lower.contains("authorization") -} - -fn cloud_provider(path: &Path) -> &'static str { - let value = path.to_string_lossy(); - if value.starts_with("s3://") { - "s3" - } else if value.starts_with("gs://") { - "gcs" - } else { - "azure" - } -} - #[cfg(test)] mod tests { use super::*; @@ -154,8 +81,8 @@ mod tests { #[test] fn returns_file_not_found_error_for_missing_file() { - let missing = PathBuf::from("/tmp/dtoo-missing-fingerprint-input.bin"); - let err = fingerprint_file(&missing).expect_err("missing file should fail"); + let err = fingerprint_file(Path::new("/tmp/dtoo-missing-fingerprint-input.bin")) + .expect_err("missing file should fail"); assert!(matches!(err, DtooError::FileNotFound { .. })); } @@ -170,4 +97,17 @@ mod tests { let name = display_name(Path::new("s3://bucket/path/to/sales.parquet")); assert_eq!(name, "sales.parquet"); } + + #[test] + fn cloud_fingerprint_is_rejected_with_clear_error() { + let err = fingerprint_file(Path::new("s3://bucket/data.parquet")) + .expect_err("cloud should error"); + match err { + DtooError::Config { message } => { + assert!(message.contains("cloud storage")); + assert!(message.contains("not supported")); + } + other => panic!("expected Config error, got {other:?}"), + } + } } diff --git a/src/inspect.rs b/src/inspect.rs index be91513..562eaea 100644 --- a/src/inspect.rs +++ b/src/inspect.rs @@ -1,12 +1,9 @@ use comfy_table::{Cell, ContentArrangement, Row, Table, presets::UTF8_FULL}; +use polars::prelude::DataType; use crate::{ - cli::InspectArgs, - crypto, - engine::{CloudSettings, DuckDbEngine, EngineConfig}, - error::DtooError, - path_utils::{is_cloud_path, split_excel_sheet_from_path}, - sql_utils::escape_sql_literal, + cli::InspectArgs, crypto, error::DtooError, path_utils::split_excel_sheet_from_path, + polars_engine::PolarsEngine, types::InputFormat, }; pub fn run(args: &InspectArgs) -> Result<(), DtooError> { @@ -14,35 +11,23 @@ pub fn run(args: &InspectArgs) -> Result<(), DtooError> { let (path, sheet) = split_excel_sheet_from_path(&target); let format = detect_format(&path)?; - let engine = DuckDbEngine::new(EngineConfig { - cloud: CloudSettings { - s3_region: args.s3_region.clone(), - s3_profile: args.s3_profile.clone(), - s3_access_key_id: None, - gcs_project_id: args.gcs_project.clone(), - azure_storage_account_name: args.azure_account.clone(), - }, - load_cloud_extensions: is_cloud_path(&path), - load_excel_extension: matches!(format, InspectFormat::Excel), - })?; - - let source_sql = build_source_sql(&path, &format, sheet.as_deref(), args.delimiter); - let row_count = engine.query_count(&format!("SELECT COUNT(*) FROM ({source_sql}) src"))?; - let schema_rows = engine.query(&format!("DESCRIBE SELECT * FROM ({source_sql}) src"))?; - let preview_rows = engine.query(&format!( - "SELECT * FROM ({source_sql}) src LIMIT {}", - args.rows - ))?; + let input_format = to_input_format(&format, sheet.as_deref(), &path, args.delimiter); + + let engine = PolarsEngine::new(); + let lf = engine.scan(&path, &input_format)?; + + let row_count = engine.row_count(lf.clone())?; + let schema = engine.schema_of(&lf)?; + let df = engine.collect(lf)?; + let preview = df.head(Some(args.rows)); if args.crypto_discover { - engine.execute("DROP TABLE IF EXISTS temp_results")?; - engine.execute(&format!("CREATE TABLE temp_results AS {source_sql}"))?; let rows = if let Some(profile_name) = args.crypto_profile.as_deref() { let profile = crypto::resolve_profile(profile_name, args.crypto_profiles_file.as_deref(), None)?; - crypto::discover_wrapped_values(&engine, &profile.detection, &profile.columns)? + crypto::discover_wrapped_values_df(&df, &profile.detection, &profile.columns)? } else { - crypto::discover_wrapped_values(&engine, &crypto::DetectionConfig::default(), &[])? + crypto::discover_wrapped_values_df(&df, &crypto::DetectionConfig::default(), &[])? }; println!(); @@ -68,18 +53,15 @@ pub fn run(args: &InspectArgs) -> Result<(), DtooError> { println!("File: {}", args.path.display()); println!("Format: {}", format_label(&format)); println!("Rows: {}", format_count(row_count)); - println!("Columns: {}", schema_rows.len()); + println!("Columns: {}", schema.len()); println!(); println!("Schema:"); - for row in &schema_rows { - let name = row.values.first().map(String::as_str).unwrap_or("?"); - let dtype = row.values.get(1).map(String::as_str).unwrap_or("?"); - let nullable = row.values.get(2).map(String::as_str).unwrap_or("?"); - println!(" {:<14} {:<14} {}", name, dtype, nullable); + for (name, dtype) in &schema { + println!(" {name:<14} {dtype}"); } println!(); println!("Preview (first {} rows):", args.rows); - println!("{}", render_preview_table(&schema_rows, &preview_rows)); + println!("{}", render_preview_table(&schema, &preview)); Ok(()) } @@ -112,13 +94,12 @@ fn detect_format(path: &str) -> Result { }) } -fn build_source_sql( - path: &str, +fn to_input_format( format: &InspectFormat, sheet: Option<&str>, + path: &str, delimiter: char, -) -> String { - let escaped = escape_sql_literal(path); +) -> InputFormat { match format { InspectFormat::Csv => { let delim = if path.to_ascii_lowercase().ends_with(".tsv") { @@ -126,48 +107,55 @@ fn build_source_sql( } else { delimiter }; - format!( - "SELECT * FROM read_csv('{escaped}', delim='{}', header=true, auto_detect=true)", - escape_sql_literal(&delim.to_string()) - ) - } - InspectFormat::Parquet => format!("SELECT * FROM read_parquet('{escaped}')"), - InspectFormat::Ndjson => format!("SELECT * FROM read_ndjson_auto('{escaped}')"), - InspectFormat::Excel => { - if let Some(chosen) = sheet { - format!( - "SELECT * FROM read_xlsx('{escaped}', sheet='{}')", - escape_sql_literal(chosen) - ) - } else { - format!("SELECT * FROM read_xlsx('{escaped}')") - } + InputFormat::Csv { delimiter: delim } } + InspectFormat::Parquet => InputFormat::Parquet, + InspectFormat::Ndjson => InputFormat::Ndjson, + InspectFormat::Excel => InputFormat::Excel { + sheet: sheet.map(ToString::to_string), + }, } } fn render_preview_table( - schema_rows: &[crate::engine::QueryRow], - preview_rows: &[crate::engine::QueryRow], + schema: &[(String, DataType)], + preview: &polars::frame::DataFrame, ) -> Table { let mut table = Table::new(); table.load_preset(UTF8_FULL); table.set_content_arrangement(ContentArrangement::Dynamic); - let header = schema_rows + let header = schema .iter() - .map(|row| Cell::new(row.values.first().cloned().unwrap_or_default())) + .map(|(name, _)| Cell::new(name)) .collect::>(); table.set_header(header); - for row in preview_rows { - table.add_row(Row::from( - row.values - .iter() - .cloned() - .map(Cell::new) - .collect::>(), - )); + // Build per-column string vectors to avoid repeated schema lookups + let col_strings: Vec>> = schema + .iter() + .map(|(name, _)| { + preview + .column(name) + .ok() + .and_then(|col| col.cast(&DataType::String).ok()) + .and_then(|col| { + col.str().ok().map(|ca| { + (0..preview.height()) + .map(|i| ca.get(i).map(|s| s.to_string())) + .collect() + }) + }) + .unwrap_or_else(|| vec![None; preview.height()]) + }) + .collect(); + + for row_idx in 0..preview.height() { + let cells: Vec = col_strings + .iter() + .map(|col| Cell::new(col.get(row_idx).and_then(|s| s.as_deref()).unwrap_or(""))) + .collect(); + table.add_row(Row::from(cells)); } table @@ -197,6 +185,7 @@ fn format_count(value: usize) -> String { #[cfg(test)] mod tests { use super::*; + use crate::path_utils::is_cloud_path; #[test] fn split_excel_sheet_parses_colon_syntax() { diff --git a/src/lineage.rs b/src/lineage.rs index 5743268..7710da0 100644 --- a/src/lineage.rs +++ b/src/lineage.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use sha2::{Digest, Sha256}; use uuid::Uuid; -use crate::{engine::DuckDbEngine, error::DtooError, sql_utils::escape_sql_literal}; +use crate::error::DtooError; #[derive(Clone, Debug, Eq, PartialEq, Hash)] enum LineageColumn { @@ -45,7 +45,6 @@ pub struct LineageManager { batch_id: String, batch_timestamp: DateTime, batch_hash: String, - origin_tracking_initialized: bool, } impl LineageManager { @@ -61,7 +60,6 @@ impl LineageManager { batch_id, batch_timestamp, batch_hash, - origin_tracking_initialized: false, }) } @@ -70,77 +68,6 @@ impl LineageManager { self.requested.contains(&LineageColumn::OriginFile) } - /// Add/update internal `_origin_file` tracking values for newly inserted rows. - pub fn tag_rows_with_origin( - &mut self, - engine: &DuckDbEngine, - path: &str, - ) -> Result<(), DtooError> { - if !self.requires_origin_tracking() { - return Ok(()); - } - - if !self.origin_tracking_initialized { - engine.execute("ALTER TABLE temp_results ADD COLUMN _origin_file VARCHAR")?; - self.origin_tracking_initialized = true; - } - - let escaped = escape_sql_literal(path); - engine.execute(&format!( - "UPDATE temp_results SET _origin_file = '{escaped}' WHERE _origin_file IS NULL" - ))?; - Ok(()) - } - - /// Apply requested lineage columns to `temp_results`. - pub fn apply_columns(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { - if self.requested.is_empty() { - return Ok(()); - } - - if self.requested.contains(&LineageColumn::BatchId) { - engine.execute(&format!( - "ALTER TABLE temp_results ADD COLUMN batch_id VARCHAR DEFAULT '{}'", - escape_sql_literal(&self.batch_id) - ))?; - } - - if self.requested.contains(&LineageColumn::RecordId) { - engine.execute("ALTER TABLE temp_results ADD COLUMN record_id VARCHAR")?; - engine.execute("UPDATE temp_results SET record_id = uuid()::VARCHAR")?; - } - - if self.requested.contains(&LineageColumn::BatchTimestamp) { - engine.execute(&format!( - "ALTER TABLE temp_results ADD COLUMN batch_timestamp TIMESTAMP DEFAULT '{}'", - escape_sql_literal(&self.batch_timestamp.to_rfc3339()) - ))?; - } - - if self.requested.contains(&LineageColumn::BatchHash) { - engine.execute(&format!( - "ALTER TABLE temp_results ADD COLUMN batch_hash VARCHAR DEFAULT '{}'", - escape_sql_literal(&self.batch_hash) - ))?; - } - - let has_internal_origin = temp_results_has_column(engine, "_origin_file")?; - if self.requested.contains(&LineageColumn::OriginFile) { - if !has_internal_origin { - return Err(DtooError::Schema { - message: - "origin_file lineage requested but internal _origin_file column is missing" - .to_string(), - }); - } - engine.execute("ALTER TABLE temp_results RENAME COLUMN _origin_file TO origin_file")?; - } else if has_internal_origin { - engine.execute("ALTER TABLE temp_results DROP COLUMN _origin_file")?; - } - - Ok(()) - } - /// Returns generated batch identifier for this run. pub fn batch_id(&self) -> &str { &self.batch_id @@ -271,18 +198,9 @@ fn compute_batch_hash(context: &LineageContext) -> String { hex::encode(hasher.finalize()) } -fn temp_results_has_column(engine: &DuckDbEngine, name: &str) -> Result { - let rows = engine.query("DESCRIBE temp_results")?; - Ok(rows - .iter() - .filter_map(|row| row.values.first()) - .any(|column| column == name)) -} - #[cfg(test)] mod tests { use super::*; - use crate::engine::{CloudSettings, EngineConfig}; #[test] fn apply_lineage_adds_requested_columns_and_renames_origin() { @@ -332,35 +250,4 @@ mod tests { let second = compute_batch_hash(&context); assert_eq!(first, second); } - - #[test] - fn apply_columns_renames_origin_file() { - let engine = DuckDbEngine::new(EngineConfig { - cloud: CloudSettings::default(), - load_cloud_extensions: false, - load_excel_extension: false, - }) - .expect("engine init"); - engine - .execute("CREATE TABLE temp_results (id INTEGER, _origin_file VARCHAR)") - .expect("create table"); - engine - .execute("INSERT INTO temp_results VALUES (1, '/tmp/a.csv')") - .expect("insert row"); - - let manager = LineageManager::new( - Some("origin_file"), - LineageContext { - files: vec!["/tmp/a.csv".to_string()], - ..LineageContext::default() - }, - ) - .expect("build manager"); - manager.apply_columns(&engine).expect("apply columns"); - - let rows = engine - .query("SELECT origin_file FROM temp_results") - .expect("query origin"); - assert_eq!(rows[0].values[0], "/tmp/a.csv"); - } } diff --git a/src/main.rs b/src/main.rs index 9919d4b..f91df65 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,8 +4,6 @@ mod config; mod convert_command; #[allow(dead_code)] mod crypto; -#[allow(dead_code)] -mod engine; mod error; #[allow(dead_code)] mod file_resolution; @@ -25,7 +23,6 @@ mod on_error; mod output_writer; #[allow(dead_code)] mod path_utils; -#[allow(dead_code)] mod polars_engine; #[allow(dead_code)] mod profile_command; @@ -38,8 +35,6 @@ mod reference_tables; #[allow(dead_code)] mod schema; #[allow(dead_code)] -mod sql_utils; -#[allow(dead_code)] mod types; use std::process::ExitCode; diff --git a/src/masking.rs b/src/masking.rs index e1ddedb..bcff990 100644 --- a/src/masking.rs +++ b/src/masking.rs @@ -3,11 +3,7 @@ use std::collections::HashSet; use polars::prelude::*; use sha2::{Digest, Sha256}; -use crate::{ - engine::DuckDbEngine, - error::DtooError, - sql_utils::{escape_sql_literal, quote_identifier}, -}; +use crate::error::DtooError; /// Replace each selected column's non-null values with `hex(sha256("{salt}:{col}:" + value))`. /// @@ -68,132 +64,9 @@ pub fn mask_dataframe( Ok(df) } -/// Applies deterministic masking updates to selected columns. -#[derive(Clone, Debug)] -pub struct MaskingEngine { - columns: Vec, - salt: String, -} - -impl MaskingEngine { - /// Build masking engine from CLI inputs. - pub fn new(mask: Option<&str>, salt: &str) -> Result { - let columns = parse_columns(mask)?; - Ok(Self { - columns, - salt: salt.to_string(), - }) - } - - /// Apply masking updates to `temp_results`. - pub fn apply(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { - if self.columns.is_empty() { - return Ok(()); - } - - let available = available_columns(engine)?; - for column in &self.columns { - if !available.contains(column) { - let mut sorted = available.iter().cloned().collect::>(); - sorted.sort(); - return Err(DtooError::Config { - message: format!( - "mask column `{column}` not found. available columns: {}", - sorted.join(", ") - ), - }); - } - - let escaped_literal = escape_sql_literal(&format!("{}:{}:", self.salt, column)); - let identifier = quote_identifier(column); - engine.execute(&format!( - "UPDATE temp_results \ - SET {identifier} = sha256('{escaped_literal}' || {identifier}::VARCHAR) \ - WHERE {identifier} IS NOT NULL" - ))?; - } - - Ok(()) - } -} - -fn parse_columns(mask: Option<&str>) -> Result, DtooError> { - let Some(mask) = mask else { - return Ok(Vec::new()); - }; - - let mut seen = HashSet::new(); - let mut columns = Vec::new(); - for column in mask.split(',').map(str::trim).filter(|v| !v.is_empty()) { - let lowered = column.to_ascii_lowercase(); - if seen.insert(lowered) { - columns.push(column.to_string()); - } - } - - if columns.is_empty() { - return Err(DtooError::Config { - message: "--mask requires at least one column name".to_string(), - }); - } - - Ok(columns) -} - -fn available_columns(engine: &DuckDbEngine) -> Result, DtooError> { - let rows = engine.query("DESCRIBE temp_results")?; - Ok(rows - .iter() - .filter_map(|row| row.values.first()) - .cloned() - .collect()) -} - #[cfg(test)] mod tests { use super::*; - use crate::engine::{CloudSettings, EngineConfig}; - - fn test_engine() -> DuckDbEngine { - DuckDbEngine::new(EngineConfig { - cloud: CloudSettings::default(), - load_cloud_extensions: false, - load_excel_extension: false, - }) - .expect("engine init") - } - - #[test] - fn masks_columns_deterministically_and_preserves_null() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (email VARCHAR)") - .expect("create table"); - engine - .execute("INSERT INTO temp_results VALUES ('a@example.com'), ('a@example.com'), (NULL)") - .expect("insert rows"); - - let masking = MaskingEngine::new(Some("email"), "project-x").expect("build masking"); - masking.apply(&engine).expect("apply masking"); - - let rows = engine - .query("SELECT email FROM temp_results ORDER BY email NULLS LAST") - .expect("query rows"); - assert_eq!(rows[0].values[0], rows[1].values[0]); - assert_eq!(rows[2].values[0], "NULL"); - } - - #[test] - fn returns_error_for_missing_column() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (email VARCHAR)") - .expect("create table"); - - let masking = MaskingEngine::new(Some("missing_col"), "").expect("build masking"); - let err = masking.apply(&engine).expect_err("should fail"); - assert!(matches!(err, DtooError::Config { .. })); - } #[test] fn mask_columns_is_deterministic_and_preserves_null() { diff --git a/src/output_writer.rs b/src/output_writer.rs index 9de96c1..2d5f4a2 100644 --- a/src/output_writer.rs +++ b/src/output_writer.rs @@ -3,13 +3,12 @@ use std::path::{Path, PathBuf}; use polars::prelude::DataFrame; use crate::{ - engine::DuckDbEngine, error::DtooError, polars_engine::PolarsEngine, types::{CompressionCodec, ExportFormat}, }; -/// Output writer configuration for exporting `temp_results`. +/// Output writer configuration for exporting the result DataFrame. #[derive(Clone, Debug, Eq, PartialEq)] pub struct OutputWriterConfig { pub output: Option, @@ -95,74 +94,6 @@ impl OutputWriter { Ok(destination) } - /// Export temp_results to configured destination via `DuckDbEngine`. - /// - /// Kept for the DuckDB-based pipeline until P2-9 switches to the DataFrame path. - /// Will be removed in P2-12. - pub fn write_from_engine(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { - let destination = self.effective_destination_path(); - let compression = self.effective_compression(&destination); - if let Some(path) = &destination - && let Some(parent) = path.parent() - && !parent.as_os_str().is_empty() - && !parent.exists() - { - return Err(DtooError::Output { - message: format!("output directory does not exist: {}", parent.display()), - }); - } - - engine.export_results( - destination.as_ref().and_then(|p| p.to_str()), - self.config.format, - self.config.header, - self.config.delimiter, - compression, - )?; - - if let (Some(original), Some(resolved)) = (&self.config.output, &destination) - && original != resolved - { - eprintln!("Info: output path adjusted to {}", resolved.display()); - } - - Ok(()) - } - - /// Write via `DuckDbEngine` and return effective destination path (`None` when stdout). - /// - /// Kept for the DuckDB-based pipeline until P2-9 switches to the DataFrame path. - /// Will be removed in P2-12. - pub fn write_and_get_destination_from_engine( - &self, - engine: &DuckDbEngine, - ) -> Result, DtooError> { - let destination = self.effective_destination_path(); - let compression = self.effective_compression(&destination); - if let Some(path) = &destination - && let Some(parent) = path.parent() - && !parent.as_os_str().is_empty() - && !parent.exists() - { - return Err(DtooError::Output { - message: format!("output directory does not exist: {}", parent.display()), - }); - } - engine.export_results( - destination.as_ref().and_then(|p| p.to_str()), - self.config.format, - self.config.header, - self.config.delimiter, - compression, - )?; - if let (Some(original), Some(resolved)) = (&self.config.output, &destination) - && original != resolved - { - eprintln!("Info: output path adjusted to {}", resolved.display()); - } - Ok(destination) - } - fn effective_destination_path(&self) -> Option { let Some(output) = &self.config.output else { return None; @@ -214,19 +145,9 @@ fn adjust_compressed_extension( #[cfg(test)] mod tests { use super::*; - use crate::engine::{CloudSettings, EngineConfig}; use polars::df; use std::fs; - fn test_engine() -> DuckDbEngine { - DuckDbEngine::new(EngineConfig { - cloud: CloudSettings::default(), - load_cloud_extensions: false, - load_excel_extension: false, - }) - .expect("engine init should succeed") - } - #[test] fn appends_gzip_extension_for_csv_when_missing() { let path = adjust_compressed_extension( @@ -354,98 +275,4 @@ mod tests { ); std::fs::remove_file(format!("{}.gz", base.to_string_lossy())).ok(); } - - // ----------------------------------------------------------------------- - // Legacy DuckDB-backed tests (kept until P2-9 pipeline switch) - // ----------------------------------------------------------------------- - - #[test] - fn write_from_engine_uses_adjusted_output_path() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (id INTEGER)") - .expect("create table should work"); - engine - .execute("INSERT INTO temp_results VALUES (1)") - .expect("insert should work"); - - let base = std::env::temp_dir().join(format!( - "dtoo-output-writer-legacy-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("clock after epoch") - .as_nanos() - )); - - let writer = OutputWriter::new(OutputWriterConfig { - output: Some(base.clone()), - format: ExportFormat::Csv, - header: true, - delimiter: ',', - compression: Some(CompressionCodec::Gzip), - }); - writer - .write_from_engine(&engine) - .expect("write should succeed"); - - let adjusted = PathBuf::from(format!("{}.gz", base.to_string_lossy())); - assert!(adjusted.exists()); - - fs::remove_file(adjusted).ok(); - } - - #[test] - fn write_from_engine_errors_when_output_directory_missing() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (id INTEGER)") - .expect("create table should work"); - - let writer = OutputWriter::new(OutputWriterConfig { - output: Some(PathBuf::from( - "/tmp/dtoo-nonexistent-subdir-legacy-12345/out.csv", - )), - format: ExportFormat::Csv, - header: true, - delimiter: ',', - compression: None, - }); - let err = writer - .write_from_engine(&engine) - .expect_err("should fail for missing dir"); - assert!(matches!(err, DtooError::Output { .. })); - } - - #[test] - fn write_and_get_destination_from_engine_returns_path() { - let engine = test_engine(); - engine - .execute("CREATE TABLE temp_results (id INTEGER)") - .expect("create table should work"); - engine - .execute("INSERT INTO temp_results VALUES (1)") - .expect("insert should work"); - let base = std::env::temp_dir().join(format!( - "dtoo-output-writer-path-legacy-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("clock after epoch") - .as_nanos() - )); - let writer = OutputWriter::new(OutputWriterConfig { - output: Some(base.clone()), - format: ExportFormat::Csv, - header: true, - delimiter: ',', - compression: Some(CompressionCodec::Gzip), - }); - let destination = writer - .write_and_get_destination_from_engine(&engine) - .expect("write should succeed"); - assert_eq!( - destination, - Some(PathBuf::from(format!("{}.gz", base.to_string_lossy()))) - ); - std::fs::remove_file(format!("{}.gz", base.to_string_lossy())).ok(); - } } diff --git a/src/polars_engine.rs b/src/polars_engine.rs index 78a6845..d5ae5e7 100644 --- a/src/polars_engine.rs +++ b/src/polars_engine.rs @@ -1,8 +1,4 @@ -//! Pure-Rust data engine built on Polars. Phase 1: built alongside the DuckDB -//! engine and not yet wired into the pipeline. See docs/specs/34-polars-engine.md. -// dead_code allowed via `#[allow(dead_code)]` on the `mod polars_engine` declaration in main.rs; -// removed in Phase 2 when the pipeline is rewired onto this engine. -#![allow(unused_imports)] +//! Pure-Rust data engine built on Polars. See docs/specs/34-polars-engine.md. use std::path::Path; diff --git a/src/profile_command.rs b/src/profile_command.rs index 1705955..e3582da 100644 --- a/src/profile_command.rs +++ b/src/profile_command.rs @@ -3,10 +3,9 @@ use std::path::PathBuf; use crate::{ cli::{OnErrorMode, PipeMode, ProfileArgs, StdinFormat}, crypto, - engine::{CloudSettings, DuckDbEngine, EngineConfig}, error::DtooError, file_resolution::{FileFormat, FileResolver, FileResolverConfig}, - path_utils::is_cloud_path, + polars_engine::PolarsEngine, profiler::{ProfileOptions, Profiler}, types::InputFormat, }; @@ -32,29 +31,16 @@ pub fn run(args: &ProfileArgs) -> Result<(), DtooError> { message: "profile requires exactly one input file".to_string(), })?; - let engine = DuckDbEngine::new(EngineConfig { - cloud: CloudSettings { - s3_region: args.s3_region.clone(), - s3_profile: args.s3_profile.clone(), - s3_access_key_id: None, - gcs_project_id: args.gcs_project.clone(), - azure_storage_account_name: args.azure_account.clone(), - }, - load_cloud_extensions: is_cloud_path(&file.path), - load_excel_extension: matches!(file.format, FileFormat::Excel { .. }), - })?; + let engine = PolarsEngine::new(); let format = to_input_format(&file.format, file.sheet.as_deref()); - - engine.register_magic_table(&file.path, &format)?; - engine.execute("DROP VIEW IF EXISTS temp_results")?; - engine.execute("DROP TABLE IF EXISTS _profile_data")?; - engine.execute("CREATE TABLE _profile_data AS SELECT * FROM _")?; - engine.execute("CREATE VIEW temp_results AS SELECT * FROM _profile_data")?; + let lf = engine.scan(&file.path, &format)?; + let mut df = engine.collect(lf)?; if let Some(profile_name) = args.crypto_profile.as_deref() { let profile = crypto::resolve_profile(profile_name, args.crypto_profiles_file.as_deref(), None)?; - let _ = crypto::decrypt_temp_results(&engine, &profile)?; + let (new_df, _) = crypto::decrypt_dataframe(df, &profile)?; + df = new_df; } let options = ProfileOptions { @@ -62,7 +48,7 @@ pub fn run(args: &ProfileArgs) -> Result<(), DtooError> { format: args.format, sample_percentage: args.sample, }; - Profiler::generate_from_engine(&engine, &options) + Profiler::generate(&df, &options) } fn to_input_format(format: &FileFormat, sheet: Option<&str>) -> InputFormat { diff --git a/src/profiler.rs b/src/profiler.rs index 09d74d7..5f3b555 100644 --- a/src/profiler.rs +++ b/src/profiler.rs @@ -8,9 +8,7 @@ use chrono::Utc; use polars::prelude::*; use serde::Serialize; -use crate::{ - cli::ProfileFormat, engine::DuckDbEngine, error::DtooError, sql_utils::quote_identifier, -}; +use crate::{cli::ProfileFormat, error::DtooError}; #[derive(Clone, Debug, Serialize)] pub struct ValueFrequency { @@ -56,7 +54,7 @@ pub struct ProfileOptions { pub sample_percentage: u8, } -/// Computes and renders profile reports from a [`DataFrame`] or from `temp_results` (legacy). +/// Computes and renders profile reports from a [`DataFrame`]. pub struct Profiler; impl Profiler { @@ -85,41 +83,6 @@ impl Profiler { let report = build_report(source, options.sample_percentage)?; write_report(options, &report) } - - /// Legacy path: compute a profile report from the DuckDB `temp_results` table. - /// - /// Retained for Task-9 compatibility; will be removed when the pipeline is - /// fully ported to Polars (Phase 2, Task 12). - pub fn generate_from_engine( - engine: &DuckDbEngine, - options: &ProfileOptions, - ) -> Result<(), DtooError> { - if options.sample_percentage == 0 || options.sample_percentage > 100 { - return Err(DtooError::Config { - message: "--profile-sample must be between 1 and 100".to_string(), - }); - } - - let sampled = options.sample_percentage < 100; - let source = if sampled { - let source = "_profile_source"; - engine.execute("DROP VIEW IF EXISTS _profile_source")?; - engine.execute(&format!( - "CREATE VIEW _profile_source AS SELECT * FROM temp_results USING SAMPLE {}%", - options.sample_percentage - ))?; - source - } else { - "temp_results" - }; - - let report_result = build_report_from_engine(engine, source, options.sample_percentage); - if sampled { - let _ = engine.execute("DROP VIEW IF EXISTS _profile_source"); - } - let report = report_result?; - write_report(options, &report) - } } // ── Polars-based report builder ─────────────────────────────────────────────── @@ -364,185 +327,6 @@ fn is_date_like_dtype(dt: &DataType) -> bool { ) } -// ── DuckDB-based report builder (legacy, kept for Task-9 bridge) ───────────── - -fn build_report_from_engine( - engine: &DuckDbEngine, - source_table: &str, - sample_percentage: u8, -) -> Result { - let row_count = engine.query_count(&format!("SELECT COUNT(*) FROM {source_table}"))?; - let mut columns = Vec::new(); - - let defs = engine.query( - "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'temp_results' ORDER BY ordinal_position", - )?; - - for row in defs { - let Some(name) = row.values.first() else { - continue; - }; - let data_type = row.values.get(1).cloned().unwrap_or_default(); - columns.push(profile_column_from_engine( - engine, - source_table, - name, - &data_type, - )?); - } - - Ok(ProfileReport { - row_count, - sample_percentage, - generated_at: Utc::now().to_rfc3339(), - columns, - }) -} - -fn profile_column_from_engine( - engine: &DuckDbEngine, - source_table: &str, - column_name: &str, - data_type: &str, -) -> Result { - let col = quote_identifier(column_name); - - let base = engine.query(&format!( - "SELECT COUNT(*), COUNT(*) - COUNT({col}), ROUND(100.0 * (COUNT(*) - COUNT({col})) / NULLIF(COUNT(*),0), 2), COUNT(DISTINCT {col}) FROM {source_table}" - ))?; - - let get = |idx: usize| -> String { - base.first() - .and_then(|r| r.values.get(idx)) - .cloned() - .unwrap_or_else(|| "0".to_string()) - }; - - let count = get(0).parse::().unwrap_or(0); - let null_count = get(1).parse::().unwrap_or(0); - let null_percentage = get(2).parse::().unwrap_or(0.0); - let distinct_count = get(3).parse::().unwrap_or(0); - - let mut profile = ColumnProfile { - name: column_name.to_string(), - data_type: data_type.to_string(), - count, - null_count, - null_percentage, - distinct_count, - min: None, - max: None, - mean: None, - stddev: None, - median: None, - p25: None, - p75: None, - min_length: None, - max_length: None, - avg_length: None, - top_5_values: top_values_from_engine(engine, source_table, &col)?, - pattern_sample: Vec::new(), - }; - - if is_numeric_str(data_type) { - let rows = engine.query(&format!( - "SELECT MIN({col}), MAX({col}), AVG({col}), STDDEV({col}), MEDIAN({col}), QUANTILE_CONT({col}, 0.25), QUANTILE_CONT({col}, 0.75) FROM {source_table}" - ))?; - profile.min = value_at_engine(&rows, 0); - profile.max = value_at_engine(&rows, 1); - profile.mean = value_at_engine(&rows, 2); - profile.stddev = value_at_engine(&rows, 3); - profile.median = value_at_engine(&rows, 4); - profile.p25 = value_at_engine(&rows, 5); - profile.p75 = value_at_engine(&rows, 6); - } else if is_text_str(data_type) { - let rows = engine.query(&format!( - "SELECT MIN(LENGTH({col})), MAX(LENGTH({col})), AVG(LENGTH({col})) FROM {source_table} WHERE {col} IS NOT NULL" - ))?; - profile.min_length = value_at_engine(&rows, 0); - profile.max_length = value_at_engine(&rows, 1); - profile.avg_length = value_at_engine(&rows, 2); - profile.pattern_sample = text_patterns_from_engine(engine, source_table, &col)?; - } else if is_date_like_str(data_type) { - let rows = engine.query(&format!( - "SELECT MIN({col}), MAX({col}) FROM {source_table}" - ))?; - profile.min = value_at_engine(&rows, 0); - profile.max = value_at_engine(&rows, 1); - } - - Ok(profile) -} - -fn top_values_from_engine( - engine: &DuckDbEngine, - source_table: &str, - col: &str, -) -> Result, DtooError> { - let rows = engine.query(&format!( - "SELECT {col}::VARCHAR, COUNT(*) FROM {source_table} WHERE {col} IS NOT NULL GROUP BY 1 ORDER BY 2 DESC LIMIT 5" - ))?; - Ok(rows - .into_iter() - .map(|row| ValueFrequency { - value: row.values.first().cloned().unwrap_or_default(), - freq: row - .values - .get(1) - .and_then(|v| v.parse::().ok()) - .unwrap_or(0), - }) - .collect()) -} - -fn text_patterns_from_engine( - engine: &DuckDbEngine, - source_table: &str, - col: &str, -) -> Result, DtooError> { - let rows = engine.query(&format!( - "SELECT regexp_replace(regexp_replace(regexp_replace({col}::VARCHAR, '[0-9]', 'd', 'g'), '[A-Za-z]', 'a', 'g'), '(d+)', 'N', 'g') AS pattern, COUNT(*) FROM {source_table} WHERE {col} IS NOT NULL GROUP BY 1 ORDER BY 2 DESC LIMIT 5" - ))?; - Ok(rows - .into_iter() - .map(|row| ValueFrequency { - value: row.values.first().cloned().unwrap_or_default(), - freq: row - .values - .get(1) - .and_then(|v| v.parse::().ok()) - .unwrap_or(0), - }) - .collect()) -} - -fn value_at_engine(rows: &[crate::engine::QueryRow], idx: usize) -> Option { - rows.first() - .and_then(|r| r.values.get(idx)) - .filter(|v| *v != "NULL") - .cloned() -} - -fn is_numeric_str(data_type: &str) -> bool { - let u = data_type.to_ascii_uppercase(); - ["INT", "DOUBLE", "FLOAT", "DECIMAL", "NUMERIC"] - .iter() - .any(|token| u.contains(token)) - && !u.contains("INTERVAL") -} - -fn is_text_str(data_type: &str) -> bool { - let u = data_type.to_ascii_uppercase(); - ["CHAR", "TEXT", "VARCHAR", "STRING"] - .iter() - .any(|token| u.contains(token)) -} - -fn is_date_like_str(data_type: &str) -> bool { - let u = data_type.to_ascii_uppercase(); - u.contains("DATE") || u.contains("TIME") -} - // ── Shared rendering (UNCHANGED) ───────────────────────────────────────────── fn write_report(options: &ProfileOptions, report: &ProfileReport) -> Result<(), DtooError> { diff --git a/src/query_pipeline.rs b/src/query_pipeline.rs index e8bbb42..11dde9a 100644 --- a/src/query_pipeline.rs +++ b/src/query_pipeline.rs @@ -1,11 +1,11 @@ use chrono::Utc; +use polars::prelude::*; use std::time::Instant; use uuid::Uuid; use crate::{ cli::{CompressMethod, OnErrorMode, OutputFormat, PipeMode, QueryArgs, StdinFormat}, crypto, - engine::{CloudSettings, DuckDbEngine, EngineConfig}, error::DtooError, file_resolution::{FileFormat, FileResolver, FileResolverConfig, ResolutionReport}, fingerprint::fingerprint_file, @@ -14,12 +14,12 @@ use crate::{ Manifest, ManifestFileDetail, ManifestFiles, ManifestOutput, ManifestTiming, ManifestWriter, build_command, duration_seconds, }, - masking::MaskingEngine, + masking::mask_dataframe, output_writer::{OutputWriter, OutputWriterConfig}, - path_utils::is_cloud_path, + polars_engine::PolarsEngine, profiler::{ProfileOptions, Profiler}, - reference_tables::{load_reference_tables, parse_reference_tables}, - schema::SchemaManager, + reference_tables::{load_reference_lazyframes, parse_reference_tables}, + schema::{SchemaManager, SchemaMode, coerce_to_schema}, types::{CompressionCodec, ExportFormat, InputFormat}, }; @@ -174,25 +174,42 @@ impl<'a> QueryPipelineRunner<'a> { } self.ensure_files_present(&files)?; - let engine = self.initialize_engine(&files)?; + let engine = PolarsEngine::new(); self.log_schema_mode(); - self.load_reference_tables(&engine)?; + let refs = self.load_reference_tables(&engine)?; let schema_manager = SchemaManager::from_schema_path(self.args.schema.as_deref())?; let mut lineage_manager = self.build_lineage_manager(&files)?; - self.process_files(&engine, &schema_manager, &mut lineage_manager, &files)?; - self.apply_crypto_stage(&engine)?; - self.apply_post_sql(&engine)?; - self.apply_masking(&engine)?; - self.apply_lineage(&engine, &lineage_manager)?; - self.apply_limit(&engine)?; + let frames = self.process_files(&engine, &mut lineage_manager, &refs, &files)?; - let row_count = self.compute_row_count_and_validate(&engine)?; + // Union-by-name accumulate, then optional explicit schema coercion. + let mut acc = engine.concat_by_name(frames)?; + if let SchemaMode::Explicit(schema) = schema_manager.mode() { + acc = coerce_to_schema(acc, &schema.columns)?; + } + let mut df = engine.collect(acc)?; + + // Crypto decrypt BEFORE post-sql (matches the original ordering). + df = self.apply_crypto_stage(df)?; + df = self.apply_post_sql(&engine, &refs, df)?; + df = self.apply_masking(df)?; + if let Some(lineage) = &self.args.lineage { + self.logger + .log(format!("Adding lineage columns: [{lineage}]")); + } + df = lineage_manager.apply_to_dataframe(df)?; + if let Some(limit) = self.args.limit { + self.logger + .log(format!("Applying limit: {}", format_count(limit))); + df = df.head(Some(limit)); + } + + let row_count = self.compute_row_count_and_validate(df.height())?; self.handle_count_output(row_count); - self.apply_output_crypto_if_needed(&engine)?; - let written_output_path = self.write_output_if_needed(&engine)?; - self.write_profile_if_requested(&engine)?; + df = self.apply_output_crypto_if_needed(df)?; + let written_output_path = self.write_output_if_needed(&engine, &df)?; + self.write_profile_if_requested(&df)?; self.fingerprint_if_requested(written_output_path)?; self.raise_partial_failure_if_needed(files.len())?; @@ -242,23 +259,6 @@ impl<'a> QueryPipelineRunner<'a> { Ok(()) } - fn initialize_engine( - &self, - files: &[crate::file_resolution::ResolvedFile], - ) -> Result { - DuckDbEngine::new(EngineConfig { - cloud: CloudSettings { - s3_region: self.args.s3_region.clone(), - s3_profile: self.args.s3_profile.clone(), - s3_access_key_id: None, - gcs_project_id: self.args.gcs_project.clone(), - azure_storage_account_name: self.args.azure_account.clone(), - }, - load_cloud_extensions: requires_cloud_extensions(self.args, files), - load_excel_extension: requires_excel_extension(self.args, files), - }) - } - fn log_schema_mode(&self) { if let Some(schema_path) = &self.args.schema { self.logger @@ -268,22 +268,20 @@ impl<'a> QueryPipelineRunner<'a> { } } - fn load_reference_tables(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { - let refs = parse_reference_tables( + fn load_reference_tables( + &self, + engine: &PolarsEngine, + ) -> Result, DtooError> { + let parsed = parse_reference_tables( &self.args.refs, self.args.delimiter, self.args.sheet.as_deref(), )?; - let loaded_refs = load_reference_tables(engine, &refs)?; - for loaded in &loaded_refs { - self.logger.log(format!( - "Loading ref table: {} ({}) — {} rows", - loaded.name, - loaded.path, - format_count(loaded.row_count) - )); + let loaded = load_reference_lazyframes(engine, &parsed)?; + for (name, _) in &loaded { + self.logger.log(format!("Loading ref table: {name}")); } - Ok(()) + Ok(loaded) } fn build_lineage_manager( @@ -312,39 +310,37 @@ impl<'a> QueryPipelineRunner<'a> { fn process_files( &mut self, - engine: &DuckDbEngine, - schema_manager: &SchemaManager, + engine: &PolarsEngine, lineage_manager: &mut LineageManager, + refs: &[(String, LazyFrame)], files: &[crate::file_resolution::ResolvedFile], - ) -> Result<(), DtooError> { + ) -> Result, DtooError> { let mut processed = 0usize; let mut skipped = 0usize; - let mut first_insert_done = false; + let track_origin = lineage_manager.requires_origin_tracking(); + let mut frames: Vec = Vec::with_capacity(files.len()); for (idx, file) in files.iter().enumerate() { let input_format = to_input_format(file.format.clone(), file.sheet.clone()); - let result = (|| -> Result { - engine.register_magic_table(&file.path, &input_format)?; - let query_sql = prepare_filter_query( + let result = (|| -> Result<(LazyFrame, usize), DtooError> { + let lf = engine.scan(&file.path, &input_format)?; + let mut lf = per_file_filter( engine, + lf, + refs, self.args.where_clause.as_deref(), self.args.filter_sql.as_deref(), )?; - - if first_insert_done { - let rows = schema_manager.insert_file_rows(engine, Some(&query_sql))?; - lineage_manager.tag_rows_with_origin(engine, &file.path)?; - Ok(rows) - } else { - let rows = schema_manager.initialize_temp_results(engine, Some(&query_sql))?; - first_insert_done = true; - lineage_manager.tag_rows_with_origin(engine, &file.path)?; - Ok(rows) + if track_origin { + lf = lf.with_column(lit(file.path.as_str()).alias("_origin_file")); } + let rows = engine.row_count(lf.clone())?; + Ok((lf, rows)) })(); match result { - Ok(rows_matched) => { + Ok((lf, rows_matched)) => { + frames.push(lf); processed += 1; self.summary.file_details.push(FileResult { path: file.path.clone(), @@ -392,28 +388,30 @@ impl<'a> QueryPipelineRunner<'a> { skipped, }); } - Ok(()) + Ok(frames) } - fn apply_post_sql(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { - if let Some(post_sql) = &self.args.post_sql { - self.logger.log("Applying post-sql..."); - engine.execute("CREATE OR REPLACE VIEW _ AS SELECT * FROM temp_results")?; - engine.execute(&format!( - "CREATE OR REPLACE TABLE temp_results AS {post_sql}" - ))?; - let post_count = engine.query_count("SELECT COUNT(*) FROM temp_results")?; - self.logger.log(format!( - "Post-sql complete: {} rows", - format_count(post_count) - )); - } - Ok(()) + fn apply_post_sql( + &self, + engine: &PolarsEngine, + refs: &[(String, LazyFrame)], + df: DataFrame, + ) -> Result { + let Some(post_sql) = &self.args.post_sql else { + return Ok(df); + }; + self.logger.log("Applying post-sql..."); + let out = engine.collect(engine.run_sql(df.lazy(), refs, post_sql)?)?; + self.logger.log(format!( + "Post-sql complete: {} rows", + format_count(out.height()) + )); + Ok(out) } - fn apply_crypto_stage(&mut self, engine: &DuckDbEngine) -> Result<(), DtooError> { + fn apply_crypto_stage(&mut self, df: DataFrame) -> Result { let Some(profile_name) = self.args.crypto_profile.as_deref() else { - return Ok(()); + return Ok(df); }; let profile = crypto::resolve_profile( @@ -421,7 +419,7 @@ impl<'a> QueryPipelineRunner<'a> { self.args.crypto_profiles_file.as_deref(), self.args.config.as_deref(), )?; - let result = crypto::decrypt_temp_results(engine, &profile)?; + let (df, result) = crypto::decrypt_dataframe(df, &profile)?; self.decrypted_columns = result.decrypted_columns; self.profile_allows_plaintext = profile.output.allow_plaintext; if !self.decrypted_columns.is_empty() { @@ -430,10 +428,10 @@ impl<'a> QueryPipelineRunner<'a> { self.decrypted_columns.join(", ") )); } - Ok(()) + Ok(df) } - fn apply_output_crypto_if_needed(&mut self, engine: &DuckDbEngine) -> Result<(), DtooError> { + fn apply_output_crypto_if_needed(&mut self, df: DataFrame) -> Result { let has_output_encrypt = self.args.encrypt_output_profile.is_some(); crypto::enforce_output_safety( &self.decrypted_columns, @@ -442,7 +440,7 @@ impl<'a> QueryPipelineRunner<'a> { )?; let Some(profile_name) = self.args.encrypt_output_profile.as_deref() else { - return Ok(()); + return Ok(df); }; let profile = crypto::resolve_profile( @@ -461,7 +459,7 @@ impl<'a> QueryPipelineRunner<'a> { self.decrypted_columns.clone() }; - crypto::encrypt_columns(engine, &profile, &columns)?; + let df = crypto::encrypt_dataframe(df, &profile, &columns)?; if !columns.is_empty() { self.logger.log(format!( "Crypto encrypt applied to columns: {}", @@ -469,45 +467,18 @@ impl<'a> QueryPipelineRunner<'a> { )); } - Ok(()) + Ok(df) } - fn apply_masking(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { + fn apply_masking(&self, df: DataFrame) -> Result { if let Some(mask) = &self.args.mask { self.logger.log(format!("Applying masking: [{mask}]")); } - let masking = MaskingEngine::new(self.args.mask.as_deref(), &self.args.mask_salt)?; - masking.apply(engine) - } - - fn apply_lineage( - &self, - engine: &DuckDbEngine, - lineage_manager: &LineageManager, - ) -> Result<(), DtooError> { - if let Some(lineage) = &self.args.lineage { - self.logger - .log(format!("Adding lineage columns: [{lineage}]")); - } - lineage_manager.apply_columns(engine) + let columns = parse_mask_columns(self.args.mask.as_deref()); + mask_dataframe(df, &columns, &self.args.mask_salt) } - fn apply_limit(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { - if let Some(limit) = self.args.limit { - self.logger - .log(format!("Applying limit: {}", format_count(limit))); - engine.execute(&format!( - "CREATE OR REPLACE TABLE temp_results AS SELECT * FROM temp_results LIMIT {limit}" - ))?; - } - Ok(()) - } - - fn compute_row_count_and_validate( - &mut self, - engine: &DuckDbEngine, - ) -> Result { - let row_count = engine.query_count("SELECT COUNT(*) FROM temp_results")?; + fn compute_row_count_and_validate(&mut self, row_count: usize) -> Result { self.summary.rows_output = row_count; self.logger.log(format!( "Accumulation complete: {} rows", @@ -541,7 +512,8 @@ impl<'a> QueryPipelineRunner<'a> { fn write_output_if_needed( &mut self, - engine: &DuckDbEngine, + engine: &PolarsEngine, + df: &DataFrame, ) -> Result, DtooError> { let should_write_output = self.args.output.is_some() || !self.args.count; if !should_write_output { @@ -557,14 +529,14 @@ impl<'a> QueryPipelineRunner<'a> { delimiter: self.args.delimiter, compression: self.args.compress.map(to_compression_codec), }); - let written_output_path = writer.write_and_get_destination_from_engine(engine)?; + let written_output_path = writer.write_and_get_destination(engine, df.clone())?; self.summary.output_path = written_output_path .as_ref() .map(|path| path.display().to_string()); Ok(written_output_path) } - fn write_profile_if_requested(&self, engine: &DuckDbEngine) -> Result<(), DtooError> { + fn write_profile_if_requested(&self, df: &DataFrame) -> Result<(), DtooError> { let Some(profile_path) = &self.args.profile else { return Ok(()); }; @@ -579,8 +551,8 @@ impl<'a> QueryPipelineRunner<'a> { profile_path.display(), profile_format_to_str(self.args.profile_format) )); - Profiler::generate_from_engine( - engine, + Profiler::generate( + df, &ProfileOptions { path: profile_path.clone(), format: self.args.profile_format, @@ -880,44 +852,6 @@ fn describe_input_source(args: &QueryArgs) -> String { "explicit paths".to_string() } -fn requires_cloud_extensions( - args: &QueryArgs, - files: &[crate::file_resolution::ResolvedFile], -) -> bool { - files.iter().any(|file| is_cloud_path(&file.path)) - || args - .refs - .iter() - .filter_map(|entry| entry.split_once('=')) - .any(|(_, path)| is_cloud_path(path.trim())) - || args - .output - .as_ref() - .is_some_and(|path| is_cloud_path(path.to_string_lossy().as_ref())) -} - -fn requires_excel_extension( - args: &QueryArgs, - files: &[crate::file_resolution::ResolvedFile], -) -> bool { - files - .iter() - .any(|file| matches!(file.format, FileFormat::Excel { .. })) - || args - .refs - .iter() - .filter_map(|entry| entry.split_once('=')) - .any(|(_, path)| is_excel_path(path.trim())) -} - -fn is_excel_path(path: &str) -> bool { - let lower = path.to_ascii_lowercase(); - lower.ends_with(".xlsx") - || lower.ends_with(".xls") - || lower.contains(".xlsx:") - || lower.contains(".xls:") -} - fn write_manifest_if_requested( args: &QueryArgs, summary: &PipelineResult, @@ -983,24 +917,45 @@ fn write_manifest_if_requested( } } -fn prepare_filter_query( - engine: &DuckDbEngine, +/// Apply the optional `--where` and `--filter-sql` stages to a single file's +/// LazyFrame, preserving the original where-before-filter ordering. +/// +/// `--where` runs first (as `SELECT * FROM _ WHERE …` with no refs available), +/// then `--filter-sql` runs against the result with reference tables registered. +fn per_file_filter( + engine: &PolarsEngine, + lf: LazyFrame, + refs: &[(String, LazyFrame)], where_clause: Option<&str>, filter_sql: Option<&str>, -) -> Result { +) -> Result { match (where_clause, filter_sql) { - (None, None) => Ok("SELECT * FROM _".to_string()), - (Some(where_sql), None) => Ok(format!("SELECT * FROM _ WHERE {where_sql}")), - (None, Some(filter)) => Ok(filter.to_string()), + (None, None) => Ok(lf), + (Some(where_sql), None) => { + engine.run_sql(lf, &[], &format!("SELECT * FROM _ WHERE {where_sql}")) + } + (None, Some(filter)) => engine.run_sql(lf, refs, filter), (Some(where_sql), Some(filter)) => { - engine.execute("DROP TABLE IF EXISTS __dtoo_source")?; - engine.execute("CREATE TABLE __dtoo_source AS SELECT * FROM _")?; - engine.execute(&format!( - "CREATE OR REPLACE VIEW _ AS SELECT * FROM __dtoo_source WHERE {where_sql}" - ))?; - Ok(filter.to_string()) + let pre = engine.run_sql(lf, &[], &format!("SELECT * FROM _ WHERE {where_sql}"))?; + engine.run_sql(pre, refs, filter) + } + } +} + +/// Parse the `--mask` comma-separated column list, mirroring the masking engine's +/// case-insensitive de-duplication so identical behavior is preserved. +fn parse_mask_columns(mask: Option<&str>) -> Vec { + let Some(mask) = mask else { + return Vec::new(); + }; + let mut seen = std::collections::HashSet::new(); + let mut columns = Vec::new(); + for column in mask.split(',').map(str::trim).filter(|v| !v.is_empty()) { + if seen.insert(column.to_ascii_lowercase()) { + columns.push(column.to_string()); } } + columns } fn to_input_format(format: FileFormat, sheet: Option) -> InputFormat { @@ -1041,42 +996,6 @@ mod tests { static UNIQUE_COUNTER: AtomicU64 = AtomicU64::new(0); - #[test] - fn applies_where_and_filter_query_order() { - let engine = DuckDbEngine::new(EngineConfig::default()).expect("engine"); - engine - .execute("CREATE VIEW _ AS SELECT 1 AS id, 10 AS value") - .expect("view create"); - - let sql = prepare_filter_query( - &engine, - Some("id = 1"), - Some("SELECT * FROM _ WHERE value = 10"), - ) - .expect("query build"); - assert_eq!(sql, "SELECT * FROM _ WHERE value = 10"); - - let rows = engine.query("SELECT * FROM _").expect("query _"); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].values, vec!["1", "10"]); - } - - #[test] - fn where_and_filter_do_not_rewrite_sql_literals_or_comments() { - let engine = DuckDbEngine::new(EngineConfig::default()).expect("engine"); - engine - .execute("CREATE VIEW _ AS SELECT 1 AS id, 'FROM _ stays' AS note") - .expect("view create"); - - let filter = "SELECT id, 'FROM _ stays' AS literal_note FROM _ -- JOIN _ in comment"; - let sql = prepare_filter_query(&engine, Some("id = 1"), Some(filter)).expect("query build"); - assert_eq!(sql, filter); - - let rows = engine.query(&sql).expect("query should run"); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].values, vec!["1", "FROM _ stays"]); - } - #[test] fn pipeline_runs_with_reference_join_and_writes_output() { let input_csv = temp_path("pipeline-input", "csv"); @@ -1732,53 +1651,6 @@ mod tests { ); } - #[test] - fn enables_extensions_for_excel_input_files() { - let args = base_query_args(); - let files = vec![crate::file_resolution::ResolvedFile { - path: "/tmp/trips.xlsx".to_string(), - format: FileFormat::Excel { sheet: None }, - sheet: None, - is_temp: false, - }]; - assert!(!requires_cloud_extensions(&args, &files)); - assert!(requires_excel_extension(&args, &files)); - } - - #[test] - fn enables_extensions_for_excel_reference_tables() { - let mut args = base_query_args(); - args.refs = vec!["zones=/tmp/zones.xlsx:Sheet1".to_string()]; - assert!(!requires_cloud_extensions(&args, &[])); - assert!(requires_excel_extension(&args, &[])); - } - - #[test] - fn does_not_enable_extensions_for_local_csv_only() { - let args = base_query_args(); - let files = vec![crate::file_resolution::ResolvedFile { - path: "/tmp/trips.csv".to_string(), - format: FileFormat::Csv { delimiter: ',' }, - sheet: None, - is_temp: false, - }]; - assert!(!requires_cloud_extensions(&args, &files)); - assert!(!requires_excel_extension(&args, &files)); - } - - #[test] - fn enables_cloud_extensions_for_s3_input() { - let args = base_query_args(); - let files = vec![crate::file_resolution::ResolvedFile { - path: "s3://bucket/trips.parquet".to_string(), - format: FileFormat::Parquet, - sheet: None, - is_temp: false, - }]; - assert!(requires_cloud_extensions(&args, &files)); - assert!(!requires_excel_extension(&args, &files)); - } - #[test] fn format_count_inserts_thousands_separators() { assert_eq!(format_count(0), "0"); diff --git a/src/reference_tables.rs b/src/reference_tables.rs index 01f73a0..2e05d71 100644 --- a/src/reference_tables.rs +++ b/src/reference_tables.rs @@ -3,7 +3,6 @@ use std::{collections::HashSet, path::Path}; use polars::prelude::LazyFrame; use crate::{ - engine::DuckDbEngine, error::DtooError, path_utils::{is_cloud_path, split_excel_sheet_from_path}, polars_engine::PolarsEngine, @@ -18,14 +17,6 @@ pub struct ReferenceTable { pub format: InputFormat, } -/// A successfully loaded reference table with row count metadata. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct LoadedReferenceTable { - pub name: String, - pub path: String, - pub row_count: usize, -} - /// Parse and validate CLI `--ref NAME=PATH` entries. pub fn parse_reference_tables( refs: &[String], @@ -80,23 +71,6 @@ pub fn parse_reference_tables( Ok(parsed) } -/// Load parsed reference tables into DuckDB and return row counts. -pub fn load_reference_tables( - engine: &DuckDbEngine, - refs: &[ReferenceTable], -) -> Result, DtooError> { - let mut loaded = Vec::with_capacity(refs.len()); - for spec in refs { - let row_count = engine.create_reference_table(&spec.name, &spec.path, &spec.format)?; - loaded.push(LoadedReferenceTable { - name: spec.name.clone(), - path: spec.path.clone(), - row_count, - }); - } - Ok(loaded) -} - /// Load each reference table as a `(name, LazyFrame)` pair via the Polars engine. pub fn load_reference_lazyframes( engine: &PolarsEngine, @@ -156,19 +130,9 @@ mod tests { }; use super::*; - use crate::engine::{CloudSettings, EngineConfig}; static UNIQUE_COUNTER: AtomicU64 = AtomicU64::new(0); - fn test_engine() -> DuckDbEngine { - DuckDbEngine::new(EngineConfig { - cloud: CloudSettings::default(), - load_cloud_extensions: false, - load_excel_extension: false, - }) - .expect("engine init should succeed") - } - #[test] fn parses_excel_colon_sheet_and_detects_format() { let excel = temp_path("refs", "xlsx"); @@ -225,36 +189,6 @@ mod tests { assert!(matches!(dup, DtooError::Config { .. })); } - #[test] - fn loads_reference_table_and_allows_join() { - let ref_csv = temp_path("regions", "csv"); - fs::write(&ref_csv, "id,region_name\n1,EMEA\n").expect("write refs csv"); - let input_csv = temp_path("input", "csv"); - fs::write(&input_csv, "region_id,value\n1,10\n").expect("write input csv"); - - let refs = vec![format!("regions={}", ref_csv.to_string_lossy())]; - let parsed = parse_reference_tables(&refs, ',', None).expect("parse refs"); - - let engine = test_engine(); - let loaded = load_reference_tables(&engine, &parsed).expect("load refs"); - assert_eq!(loaded[0].row_count, 1); - - engine - .register_magic_table( - input_csv.to_string_lossy().as_ref(), - &InputFormat::Csv { delimiter: ',' }, - ) - .expect("register input"); - let rows = engine - .query("SELECT _.value, r.region_name FROM _ JOIN regions r ON _.region_id = r.id") - .expect("join should succeed"); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].values, vec!["10", "EMEA"]); - - let _ = fs::remove_file(ref_csv); - let _ = fs::remove_file(input_csv); - } - fn temp_path(prefix: &str, ext: &str) -> PathBuf { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/src/schema.rs b/src/schema.rs index cf95fe8..f3e9cbc 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -7,11 +7,9 @@ use std::{ use polars::prelude::{DataType, Expr, LazyFrame, LiteralValue, PlSmallStr, TimeUnit, col, lit}; use serde::Deserialize; -use crate::{ - engine::DuckDbEngine, error::DtooError, sql_utils::quote_identifier, types::SchemaColumn, -}; +use crate::{error::DtooError, types::SchemaColumn}; -/// Handles auto-detected or explicit schema setup for `temp_results`. +/// Handles auto-detected or explicit schema setup for the result set. #[derive(Debug, Clone)] pub struct SchemaManager { mode: SchemaMode, @@ -59,43 +57,6 @@ impl SchemaManager { pub fn mode(&self) -> &SchemaMode { &self.mode } - - /// Initialize `temp_results` and insert first-file rows. - pub fn initialize_temp_results( - &self, - engine: &DuckDbEngine, - filter_sql: Option<&str>, - ) -> Result { - let query_sql = filter_sql.unwrap_or("SELECT * FROM _"); - - match &self.mode { - SchemaMode::Auto => { - engine.create_temp_results_schema_from_query(query_sql)?; - engine.insert_into_temp_results_by_name(query_sql) - } - SchemaMode::Explicit(schema) => { - engine.create_temp_results_from_schema(&schema.columns)?; - let projected = projected_query_for_schema(engine, query_sql, &schema.columns)?; - engine.insert_into_temp_results_by_name(&projected) - } - } - } - - /// Insert subsequent-file rows with schema behavior already established. - pub fn insert_file_rows( - &self, - engine: &DuckDbEngine, - filter_sql: Option<&str>, - ) -> Result { - let query_sql = filter_sql.unwrap_or("SELECT * FROM _"); - match &self.mode { - SchemaMode::Auto => engine.merge_into_temp_results_by_name(query_sql), - SchemaMode::Explicit(schema) => { - let projected = projected_query_for_schema(engine, query_sql, &schema.columns)?; - engine.insert_into_temp_results_by_name(&projected) - } - } - } } /// Map a DuckDB-style type string to a Polars [`DataType`]. @@ -177,48 +138,6 @@ pub fn coerce_to_schema(lf: LazyFrame, columns: &[SchemaColumn]) -> Result Result { - let available = source_column_lookup(engine, source_query)?; - let select_list = columns - .iter() - .map(|column| { - if let Some(source_name) = available.get(&column.name.to_ascii_lowercase()) { - format!( - "{} AS {}", - quote_identifier(source_name), - quote_identifier(&column.name) - ) - } else { - format!("NULL AS {}", quote_identifier(&column.name)) - } - }) - .collect::>() - .join(", "); - - Ok(format!("SELECT {select_list} FROM ({source_query}) q")) -} - -fn source_column_lookup( - engine: &DuckDbEngine, - source_query: &str, -) -> Result, DtooError> { - let describe_sql = format!("DESCRIBE SELECT * FROM ({source_query}) q"); - let rows = engine.query(&describe_sql)?; - let mut columns = HashMap::with_capacity(rows.len()); - - for row in rows { - if let Some(name) = row.values.first() { - columns.insert(name.to_ascii_lowercase(), name.clone()); - } - } - - Ok(columns) -} - fn load_explicit_schema(path: &Path) -> Result { if !path.exists() { return Err(DtooError::Config { @@ -291,108 +210,10 @@ fn is_valid_duckdb_identifier(name: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::engine::{CloudSettings, EngineConfig}; - use crate::types::InputFormat; use std::sync::atomic::{AtomicU64, Ordering}; static UNIQUE_COUNTER: AtomicU64 = AtomicU64::new(0); - fn test_engine() -> DuckDbEngine { - DuckDbEngine::new(EngineConfig { - cloud: CloudSettings::default(), - load_cloud_extensions: false, - load_excel_extension: false, - }) - .expect("engine init should succeed") - } - - #[test] - fn auto_mode_initializes_from_first_file_and_inserts_subsequent() { - let engine = test_engine(); - let csv1 = temp_csv("id,name\n1,alice\n"); - let csv2 = temp_csv("name,id,extra\nbob,2,x\n"); - - engine - .register_magic_table(&csv1, &InputFormat::Csv { delimiter: ',' }) - .expect("register first file"); - - let manager = SchemaManager::from_schema_path(None).expect("auto mode should build"); - let first_rows = manager - .initialize_temp_results(&engine, None) - .expect("first insert should work"); - assert_eq!(first_rows, 1); - - engine - .register_magic_table(&csv2, &InputFormat::Csv { delimiter: ',' }) - .expect("register second file"); - let second_rows = manager - .insert_file_rows(&engine, None) - .expect("second insert should work"); - assert_eq!(second_rows, 1); - - let rows = engine - .query("SELECT id, name FROM temp_results ORDER BY id") - .expect("query should work"); - assert_eq!(rows.len(), 2); - - let _ = fs::remove_file(csv1); - let _ = fs::remove_file(csv2); - } - - #[test] - fn explicit_schema_parses_yaml_and_inserts() { - let schema_file = temp_schema( - "columns:\n - name: id\n type: INTEGER\n - name: name\n type: VARCHAR\n", - ); - - let engine = test_engine(); - let csv = temp_csv("id,name,ignored\n1,alice,x\n"); - engine - .register_magic_table(&csv, &InputFormat::Csv { delimiter: ',' }) - .expect("register file"); - - let manager = SchemaManager::from_schema_path(Some(&schema_file)).expect("schema parse"); - let inserted = manager - .initialize_temp_results(&engine, None) - .expect("insert should work"); - assert_eq!(inserted, 1); - - let rows = engine - .query("SELECT id, name FROM temp_results") - .expect("query should work"); - assert_eq!(rows[0].values, vec!["1", "alice"]); - - let _ = fs::remove_file(schema_file); - let _ = fs::remove_file(csv); - } - - #[test] - fn explicit_schema_missing_source_column_inserts_null() { - let schema_file = temp_schema( - "columns:\n - name: id\n type: INTEGER\n - name: name\n type: VARCHAR\n", - ); - - let engine = test_engine(); - let csv = temp_csv("id\n1\n"); - engine - .register_magic_table(&csv, &InputFormat::Csv { delimiter: ',' }) - .expect("register file"); - - let manager = SchemaManager::from_schema_path(Some(&schema_file)).expect("schema parse"); - let inserted = manager - .initialize_temp_results(&engine, None) - .expect("insert should work"); - assert_eq!(inserted, 1); - - let rows = engine - .query("SELECT id, name FROM temp_results") - .expect("query should work"); - assert_eq!(rows[0].values, vec!["1", "NULL"]); - - let _ = fs::remove_file(schema_file); - let _ = fs::remove_file(csv); - } - #[test] fn invalid_yaml_returns_config_error() { let schema_file = temp_schema("columns: [\n"); @@ -419,25 +240,6 @@ mod tests { let _ = fs::remove_file(schema_file); } - #[test] - fn invalid_duckdb_type_fails_at_table_creation() { - let schema_file = temp_schema("columns:\n - name: id\n type: NOPE_TYPE\n"); - let engine = test_engine(); - let manager = SchemaManager::from_schema_path(Some(&schema_file)).expect("schema parse"); - let csv = temp_csv("id\n1\n"); - engine - .register_magic_table(&csv, &InputFormat::Csv { delimiter: ',' }) - .expect("register file"); - - let err = manager - .initialize_temp_results(&engine, None) - .expect_err("invalid type should fail"); - assert!(matches!(err, DtooError::Sql { .. })); - - let _ = fs::remove_file(schema_file); - let _ = fs::remove_file(csv); - } - #[test] fn duckdb_type_maps_to_polars() { use polars::prelude::*; @@ -519,12 +321,6 @@ mod tests { path } - fn temp_csv(contents: &str) -> String { - let path = std::env::temp_dir().join(format!("dtoo-schema-{}.csv", unique_suffix())); - fs::write(&path, contents).expect("write csv file"); - path.to_string_lossy().to_string() - } - fn unique_suffix() -> String { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/src/sql_utils.rs b/src/sql_utils.rs deleted file mode 100644 index 6b6aedf..0000000 --- a/src/sql_utils.rs +++ /dev/null @@ -1,24 +0,0 @@ -/// Escape a value for safe embedding in a single-quoted SQL literal. -pub fn escape_sql_literal(input: &str) -> String { - input.replace('\'', "''") -} - -/// Quote an identifier using DuckDB-compatible double-quote escaping. -pub fn quote_identifier(input: &str) -> String { - format!("\"{}\"", input.replace('"', "\"\"")) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn escapes_single_quotes() { - assert_eq!(escape_sql_literal("o'hare"), "o''hare"); - } - - #[test] - fn quotes_identifiers() { - assert_eq!(quote_identifier("weird\"name"), "\"weird\"\"name\""); - } -} diff --git a/src/types.rs b/src/types.rs index e3ed4d3..f1bb056 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,6 +1,6 @@ //! Engine-agnostic data types shared across the dtoo pipeline. -/// Input file format for registering the magic `_` view. +/// Input file format for scanning source files. #[derive(Clone, Debug, Eq, PartialEq)] pub enum InputFormat { Parquet, @@ -9,7 +9,7 @@ pub enum InputFormat { Excel { sheet: Option }, } -/// Export format for writing `temp_results`. +/// Export format for writing the result DataFrame. #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum ExportFormat { Csv, @@ -24,7 +24,7 @@ pub enum CompressionCodec { Zstd, } -/// One explicit schema column used to build `temp_results`. +/// One explicit schema column from a user-provided schema file. #[derive(Clone, Debug, Eq, PartialEq)] pub struct SchemaColumn { pub name: String,