From 4f25585d299a97a5843797e62d32c47165526c0f Mon Sep 17 00:00:00 2001 From: joefrost01 Date: Sat, 11 Apr 2026 21:38:07 +0100 Subject: [PATCH] feat: 20 - implement inspect command --- Cargo.lock | 111 ++++++++++++++++++++++++++- Cargo.toml | 1 + src/cli.rs | 16 ++++ src/inspect.rs | 204 +++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 5 +- 5 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 src/inspect.rs diff --git a/Cargo.lock b/Cargo.lock index 2a89b2c..bb089b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -493,6 +493,7 @@ version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ + "crossterm", "unicode-segmentation", "unicode-width", ] @@ -554,6 +555,29 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags", + "crossterm_winapi", + "document-features", + "parking_lot", + "rustix", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -602,12 +626,22 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dtoo" version = "0.1.0" dependencies = [ "chrono", "clap", + "comfy-table", "duckdb", "glob", "hex", @@ -1296,7 +1330,7 @@ dependencies = [ "bitflags", "libc", "plain", - "redox_syscall", + "redox_syscall 0.7.4", ] [[package]] @@ -1311,6 +1345,21 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" @@ -1406,6 +1455,29 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1643,6 +1715,15 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.7.4" @@ -1856,6 +1937,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "seahash" version = "4.1.0" @@ -2504,6 +2591,28 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/Cargo.toml b/Cargo.toml index 6daef5a..3336fcd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,3 +16,4 @@ sha2 = "0.10" chrono = { version = "0.4", features = ["serde"] } hex = "0.4" indicatif = "0.17" +comfy-table = "7" diff --git a/src/cli.rs b/src/cli.rs index 006dba4..9dbdc19 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -162,6 +162,18 @@ pub struct InspectArgs { #[arg(long, default_value = ",")] pub delimiter: char, + + #[arg(long = "s3-region")] + pub s3_region: Option, + + #[arg(long = "s3-profile")] + pub s3_profile: Option, + + #[arg(long = "gcs-project")] + pub gcs_project: Option, + + #[arg(long = "azure-account")] + pub azure_account: Option, } /// Arguments for `dtoo fingerprint`. @@ -637,6 +649,10 @@ mod tests { assert_eq!(args.path, PathBuf::from("input.csv")); assert_eq!(args.rows, 10); assert_eq!(args.delimiter, ','); + assert_eq!(args.s3_region, None); + assert_eq!(args.s3_profile, None); + assert_eq!(args.gcs_project, None); + assert_eq!(args.azure_account, None); } _ => panic!("expected inspect command"), } diff --git a/src/inspect.rs b/src/inspect.rs new file mode 100644 index 0000000..f8c7635 --- /dev/null +++ b/src/inspect.rs @@ -0,0 +1,204 @@ +use comfy_table::{Cell, ContentArrangement, Row, Table, presets::UTF8_FULL}; + +use crate::{ + cli::InspectArgs, + engine::{CloudSettings, DuckDbEngine, EngineConfig}, + error::DtooError, +}; + +pub fn run(args: &InspectArgs) -> Result<(), DtooError> { + let target = args.path.to_string_lossy(); + let (path, sheet) = split_excel_sheet(&target); + let format = detect_format(&path)?; + + let engine = DuckDbEngine::new(EngineConfig { + cloud: CloudSettings { + s3_region: args.s3_region.clone(), + s3_profile: args.s3_profile.clone(), + s3_access_key_id: None, + gcs_project_id: args.gcs_project.clone(), + azure_storage_account_name: args.azure_account.clone(), + }, + load_extensions: is_cloud_path(&path), + })?; + + let source_sql = build_source_sql(&path, &format, sheet.as_deref(), args.delimiter); + let row_count = engine.query_count(&format!("SELECT COUNT(*) FROM ({source_sql}) src"))?; + let schema_rows = engine.query(&format!("DESCRIBE SELECT * FROM ({source_sql}) src"))?; + let preview_rows = engine.query(&format!( + "SELECT * FROM ({source_sql}) src LIMIT {}", + args.rows + ))?; + + println!("File: {}", args.path.display()); + println!("Format: {}", format_label(&format)); + println!("Rows: {}", format_count(row_count)); + println!("Columns: {}", schema_rows.len()); + println!(); + println!("Schema:"); + for row in &schema_rows { + let name = row.values.first().map(String::as_str).unwrap_or("?"); + let dtype = row.values.get(1).map(String::as_str).unwrap_or("?"); + let nullable = row.values.get(2).map(String::as_str).unwrap_or("?"); + println!(" {:<14} {:<14} {}", name, dtype, nullable); + } + println!(); + println!("Preview (first {} rows):", args.rows); + println!("{}", render_preview_table(&schema_rows, &preview_rows)); + + Ok(()) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum InspectFormat { + Csv, + Parquet, + Ndjson, + Excel, +} + +fn detect_format(path: &str) -> Result { + let lower = path.to_ascii_lowercase(); + if lower.ends_with(".csv") || lower.ends_with(".tsv") { + return Ok(InspectFormat::Csv); + } + if lower.ends_with(".parquet") { + return Ok(InspectFormat::Parquet); + } + if lower.ends_with(".ndjson") || lower.ends_with(".jsonl") { + return Ok(InspectFormat::Ndjson); + } + if lower.ends_with(".xlsx") || lower.ends_with(".xls") { + return Ok(InspectFormat::Excel); + } + + Err(DtooError::Config { + message: format!("unsupported inspect file format for path `{path}`"), + }) +} + +fn build_source_sql( + path: &str, + format: &InspectFormat, + sheet: Option<&str>, + delimiter: char, +) -> String { + let escaped = escape_sql_literal(path); + match format { + InspectFormat::Csv => { + let delim = if path.to_ascii_lowercase().ends_with(".tsv") { + '\t' + } else { + delimiter + }; + format!( + "SELECT * FROM read_csv('{escaped}', delim='{}', header=true, auto_detect=true)", + escape_sql_literal(&delim.to_string()) + ) + } + InspectFormat::Parquet => format!("SELECT * FROM read_parquet('{escaped}')"), + InspectFormat::Ndjson => format!("SELECT * FROM read_ndjson_auto('{escaped}')"), + InspectFormat::Excel => { + let chosen = sheet.unwrap_or("Sheet1"); + format!( + "SELECT * FROM st_read('{escaped}', layer='{}')", + escape_sql_literal(chosen) + ) + } + } +} + +fn render_preview_table( + schema_rows: &[crate::engine::QueryRow], + preview_rows: &[crate::engine::QueryRow], +) -> Table { + let mut table = Table::new(); + table.load_preset(UTF8_FULL); + table.set_content_arrangement(ContentArrangement::Dynamic); + + let header = schema_rows + .iter() + .map(|row| Cell::new(row.values.first().cloned().unwrap_or_default())) + .collect::>(); + table.set_header(header); + + for row in preview_rows { + table.add_row(Row::from( + row.values + .iter() + .cloned() + .map(Cell::new) + .collect::>(), + )); + } + + table +} + +fn split_excel_sheet(path: &str) -> (String, Option) { + let Some(colon_idx) = path.rfind(':') else { + return (path.to_string(), None); + }; + let (left, right_with_colon) = path.split_at(colon_idx); + let sheet = right_with_colon.trim_start_matches(':').trim(); + let left_lower = left.to_ascii_lowercase(); + if sheet.is_empty() || !(left_lower.ends_with(".xlsx") || left_lower.ends_with(".xls")) { + return (path.to_string(), None); + } + (left.to_string(), Some(sheet.to_string())) +} + +fn format_label(format: &InspectFormat) -> &'static str { + match format { + InspectFormat::Csv => "CSV", + InspectFormat::Parquet => "Parquet", + InspectFormat::Ndjson => "NDJSON", + InspectFormat::Excel => "Excel", + } +} + +fn is_cloud_path(path: &str) -> bool { + path.starts_with("s3://") + || path.starts_with("gs://") + || path.starts_with("az://") + || path.starts_with("abfss://") +} + +fn escape_sql_literal(input: &str) -> String { + input.replace('\'', "''") +} + +fn format_count(value: usize) -> String { + let digits = value.to_string(); + let mut out = String::with_capacity(digits.len() + digits.len() / 3); + for (idx, ch) in digits.chars().enumerate() { + if idx > 0 && (digits.len() - idx).is_multiple_of(3) { + out.push(','); + } + out.push(ch); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_excel_sheet_parses_colon_syntax() { + let (path, sheet) = split_excel_sheet("sales.xlsx:Sheet2"); + assert_eq!(path, "sales.xlsx"); + assert_eq!(sheet.as_deref(), Some("Sheet2")); + } + + #[test] + fn detect_format_rejects_unknown_extension() { + let err = detect_format("data.unknown").expect_err("must reject unsupported format"); + assert!(err.to_string().contains("unsupported inspect file format")); + } + + #[test] + fn cloud_path_supports_abfss() { + assert!(is_cloud_path("abfss://container/data.parquet")); + } +} diff --git a/src/main.rs b/src/main.rs index 2cde342..c50e61d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,8 @@ mod file_resolution; #[allow(dead_code)] mod fingerprint; #[allow(dead_code)] +mod inspect; +#[allow(dead_code)] mod lineage; #[allow(dead_code)] mod manifest; @@ -31,6 +33,7 @@ use std::process::ExitCode; use cli::{Cli, Commands, QueryArgs}; use error::DtooError; use fingerprint::fingerprint_file; +use inspect::run as run_inspect; use query_pipeline::QueryPipeline; fn main() -> ExitCode { @@ -63,7 +66,7 @@ fn dispatch(cli: Cli) -> Result<(), DtooError> { match cli.command { Commands::Query(args) => QueryPipeline::run(&args), Commands::Profile(_args) => Ok(()), - Commands::Inspect(_args) => Ok(()), + Commands::Inspect(args) => run_inspect(&args), Commands::Fingerprint(args) => { let hash = fingerprint_file(&args.path)?; println!("{hash} {}", args.path.display());