diff --git a/Cargo.lock b/Cargo.lock index 5d72ad4fb..cac5ef273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3612,6 +3612,27 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777d18aa0f12f8b285331cd867133ee14422b3f023f6d388034c47d43e28786a" +dependencies = [ + "approx", + "num-traits", + "serde", + "thiserror 2.0.20", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -4043,7 +4064,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" dependencies = [ "aes-gcm", "anyhow", @@ -4051,6 +4072,7 @@ dependencies = [ "array-init", "arrow-arith", "arrow-array", + "arrow-avro", "arrow-buffer", "arrow-cast", "arrow-ord", @@ -4078,6 +4100,7 @@ dependencies = [ "once_cell", "ordered-float", "parquet", + "parquet-geospatial", "rand 0.9.5", "reqwest 0.12.28", "roaring", @@ -4119,7 +4142,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" dependencies = [ "proc-macro2", "quote", @@ -4129,7 +4152,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" dependencies = [ "anyhow", "async-trait", @@ -5118,6 +5141,28 @@ dependencies = [ "libm", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num_threads" version = "0.1.7" @@ -5567,6 +5612,7 @@ dependencies = [ "num-integer", "num-traits", "object_store", + "parquet-geospatial", "ring", "seq-macro", "simdutf8", @@ -5576,6 +5622,19 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "parquet-geospatial" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634827365851180ca26bf54c76e970dc1c14c075684c0580ee97d8d0c953faf1" +dependencies = [ + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "wkb", +] + [[package]] name = "pbkdf2" version = "0.12.2" @@ -5912,6 +5971,15 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -7821,6 +7889,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.19+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd43688c4858d7264ae1cf9a9f874a67c3feb74a883b1cd7bebd6a4ea8307218" +dependencies = [ + "indexmap 2.14.2", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "tonic" version = "0.14.6" @@ -8642,12 +8740,33 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wkb" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04ab2f2845abe967a213998446345a1a6fc9a8f318cd88f668a8e507861d13bc" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "writeable" version = "0.6.4" diff --git a/Cargo.toml b/Cargo.toml index 2ef245005..48191eeed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,11 @@ resolver = "3" datafusion = { version = "55", default-features = false } datafusion-proto = { version = "55", default-features = false } +# datafusion-iceberg depends on apache/iceberg-rust; use the same source so FileScanTask is one type. +[patch."https://github.com/apache/iceberg-rust"] +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } + [workspace.lints.clippy] disallowed_types = "deny" disallowed-methods = "deny" diff --git a/datafusion/distributed-iceberg-test-tools/Cargo.toml b/datafusion/distributed-iceberg-test-tools/Cargo.toml index c39d7ae69..5fa441099 100644 --- a/datafusion/distributed-iceberg-test-tools/Cargo.toml +++ b/datafusion/distributed-iceberg-test-tools/Cargo.toml @@ -14,8 +14,8 @@ datafusion = { workspace = true, features = ["parquet", "sql"] } # The test harness runs distributed queries on in-memory workers. datafusion-distributed = { path = "../distributed", features = ["integration"] } datafusion-proto = { workspace = true } -# Match datafusion-distributed-iceberg's iceberg-rust revision so tables share one type. -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +# Match datafusion-distributed-iceberg's iceberg-rust source so tables share one type. +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } rand = { version = "0.10", default-features = false, features = ["alloc"] } rand_pcg = "0.10" tempfile = "3.18" diff --git a/datafusion/distributed-iceberg-test-tools/src/harness.rs b/datafusion/distributed-iceberg-test-tools/src/harness.rs index 57eee91c6..7b762534d 100644 --- a/datafusion/distributed-iceberg-test-tools/src/harness.rs +++ b/datafusion/distributed-iceberg-test-tools/src/harness.rs @@ -20,7 +20,10 @@ use datafusion_distributed::{ use datafusion_proto::physical_plan::AsExecutionPlan; use datafusion_proto::protobuf::PhysicalPlanNode; use iceberg::MetadataLocation; -use iceberg::spec::{MAIN_BRANCH, TableMetadata, TableMetadataBuilder}; +use iceberg::spec::{ + DataFile, MAIN_BRANCH, ManifestListWriter, ManifestWriterBuilder, Operation, Snapshot, Summary, + TableMetadata, TableMetadataBuilder, +}; use crate::FixtureTable; use crate::error::df_err; @@ -117,6 +120,7 @@ pub struct IcebergTestHarnessBuilder { table_options: BTreeMap, table: Option, metadata: Option, + data_files: Option>, workers: Option, } @@ -132,6 +136,7 @@ impl Default for IcebergTestHarnessBuilder { table_options: BTreeMap::new(), table: None, metadata: None, + data_files: None, workers: None, } } @@ -191,6 +196,16 @@ impl IcebergTestHarnessBuilder { self } + /// Replaces the current snapshot's manifest with synthetic file entries, creating a + /// snapshot if needed. Data files are not written and the snapshot's summary totals are + /// preserved, so tests can tell summary statistics from per-file manifest metrics. + /// The entries' paths are never opened, so planning-only tests can describe files, + /// metrics and bounds directly. It applies after [`Self::with_metadata`]'s transform. + pub fn with_data_files(mut self, files: impl IntoIterator) -> Self { + self.data_files = Some(files.into_iter().collect()); + self + } + /// Enables distributed planning with logical workers backed by in-memory gRPC. pub fn with_workers(mut self, workers: usize) -> Self { self.workers = Some(workers); @@ -210,24 +225,34 @@ impl IcebergTestHarnessBuilder { harness.query_with_metrics(sql).await } - /// Registers the table, if any, from customized metadata if [`Self::with_metadata`] was - /// given. + /// Registers the table, if any, from customized metadata if [`Self::with_metadata`] or + /// [`Self::with_data_files`] was given. pub async fn build(mut self) -> Result { - let table = match (self.table.take(), self.metadata.take()) { - (None, None) => None, - (None, Some(_)) => { + let table = match ( + self.table.take(), + self.metadata.take(), + self.data_files.take(), + ) { + (None, None, None) => None, + (None, _, _) => { return exec_err!("table metadata can only be customized for a table"); } - (Some(table), None) => { + (Some(table), None, None) => { let location = table.metadata_location()?.to_string(); Some((table, location)) } - (Some(table), Some(metadata)) => { - let metadata = metadata(table.table().metadata()); + (Some(table), metadata, data_files) => { + let mut customized = match metadata { + Some(metadata) => metadata(table.table().metadata()), + None => table.table().metadata().clone(), + }; + if let Some(files) = data_files { + customized = replace_manifest_with(&table, customized, files).await?; + } let location = MetadataLocation::from_str(table.metadata_location()?) .map_err(df_err)? .with_next_version(); - metadata + customized .write_to(table.table().file_io(), &location) .await .map_err(df_err)?; @@ -297,6 +322,77 @@ pub fn data_file_name(path: &str) -> &str { path.rsplit_once("/data/").map_or(path, |(_, name)| name) } +/// Writes a manifest of `files` and points the current snapshot's manifest list at it, +/// creating a snapshot when the metadata has none. Snapshot identity and summary are kept. +async fn replace_manifest_with( + table: &FixtureTable, + metadata: TableMetadata, + files: Vec, +) -> Result { + let metadata = if metadata.current_snapshot().is_none() { + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_sequence_number(1) + .with_timestamp_ms(metadata.last_updated_ms() + 1) + .with_manifest_list(format!( + "{}/metadata/synthetic-manifest-list.avro", + metadata.location() + )) + .with_schema_id(metadata.current_schema_id()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: Default::default(), + }) + .build(); + metadata + .into_builder(None) + .set_branch_snapshot(snapshot, MAIN_BRANCH) + .map_err(df_err)? + .build() + .map_err(df_err)? + .metadata + } else { + metadata + }; + let snapshot = metadata + .current_snapshot() + .expect("snapshot initialized above"); + let file_io = table.table().file_io(); + let manifest_path = format!("{}/metadata/synthetic-manifest.avro", metadata.location()); + for path in [manifest_path.as_str(), snapshot.manifest_list()] { + if file_io.exists(path).await.map_err(df_err)? { + file_io.delete(path).await.map_err(df_err)?; + } + } + let mut writer = ManifestWriterBuilder::new( + file_io.new_output(&manifest_path).map_err(df_err)?, + Some(snapshot.snapshot_id()), + Arc::clone(metadata.current_schema()), + metadata.default_partition_spec().as_ref().clone(), + ) + .build_v2_data(); + for file in files { + writer + .add_file(file, snapshot.sequence_number()) + .map_err(df_err)?; + } + let manifest = writer.write_manifest_file().await.map_err(df_err)?; + let mut list = ManifestListWriter::v2( + file_io + .new_output(snapshot.manifest_list()) + .map_err(df_err)? + .writer() + .await + .map_err(df_err)?, + snapshot.snapshot_id(), + snapshot.parent_snapshot_id(), + snapshot.sequence_number(), + ); + list.add_manifests([manifest].into_iter()).map_err(df_err)?; + list.close().await.map_err(df_err)?; + Ok(metadata) +} + /// Starts from `metadata` with its schema, partition spec and properties, but no snapshots. pub fn without_snapshots(metadata: &TableMetadata) -> TableMetadataBuilder { let snapshot_ids: Vec<_> = metadata.snapshots().map(|s| s.snapshot_id()).collect(); diff --git a/datafusion/distributed-iceberg/Cargo.toml b/datafusion/distributed-iceberg/Cargo.toml index 31e11fe37..7b355e1e3 100644 --- a/datafusion/distributed-iceberg/Cargo.toml +++ b/datafusion/distributed-iceberg/Cargo.toml @@ -12,11 +12,13 @@ async-trait = "0.1" bytes = "1" datafusion = { workspace = true, features = ["sql"] } datafusion-proto = { workspace = true } -# Match datafusion-iceberg's iceberg-rust revision so FileScanTask has one shared type. +# DataDog's iceberg-rust build of upstream main (Arrow 59), which exposes per-file column statistics on +# FileScanTask (apache/iceberg-rust#3424). The workspace [patch] makes datafusion-iceberg use the same +# source so FileScanTask is one shared type. iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg.git", rev = "20327db58c7a44330366da00a34a26e866d0da4d" } futures = "0.3" -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } -iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } datafusion-distributed = { path = "../distributed", features = ["sql"] } delegate = "0.13" prost = "0.14.1" diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index 9c5b6507b..7bb6c819d 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -84,10 +84,9 @@ reading and deliver descriptors through the coordinator's feed. Eager scans derive row counts and bytes from selected files and cap task estimates by the number of selected files. Predicates or deletes make row counts inexact; -empty selections have exactly zero rows. Whole-snapshot column statistics are -skipped when they would describe a different row set. Lazy scans retain snapshot -statistics and task-size estimates. Selected-file column statistics remain a -separate change. Task-local eager variants and decoded plans report unknown +empty selections have exactly zero rows. When column statistics are enabled, +eager scans aggregate selected-file metrics. Lazy scans retain snapshot statistics +and task-size estimates. Task-local eager variants and decoded plans report unknown statistics rather than applying whole-snapshot aggregates to a subset. Both scans keep the filters DataFusion pushes into them, including the dynamic @@ -98,6 +97,27 @@ filter. Enable `iceberg.dynamic_filtering_enabled` to make Iceberg scans consume dynamic filters pushed into them. With it on, `EXPLAIN` shows them on the scan as `dynamic_filters=`. Scans don't skip data with these filters yet. +## Selected-file statistics + +With `iceberg.column_stats_enabled=true`, eager scans reconstruct column bytes, +min/max bounds, and null counts from the **selected files**. Missing metrics remain +unknown. Bounds are inexact; null counts also become inexact when predicates or +deletes may remove rows. Nested column bytes sum leaf sizes without inventing +parent bounds or null counts. + +Snapshot distinct-count estimates are retained as an inexact fallback, capped by +selected-file row counts when known. They are not recomputed for the selected +files and can overestimate the filtered result. Sum statistics remain unknown. + +The metrics come from the planned file tasks themselves: the scan asks Iceberg to +retain the statistics columns' metrics (`TableScanBuilder::with_column_stats`), so no +manifest is read a second time. Scan task counts continue to use whole-file bytes. + +The retained metrics stay on each task, so they are also serialized to workers with the +file list (a few small per-column maps per file) and counted by the planning memory +reservation. iceberg-rust has no way to drop them from a planned task, which would +avoid that cost. + ```bash cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools ``` diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 71eebfe19..527f6665a 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -244,3 +244,13 @@ pub(crate) fn stats_from_snapshot( column_statistics: vec![ColumnStatistics::new_unknown(); schema.fields().len()], }) } + +/// Use projected compressed bytes when all column sizes are known, otherwise retain file bytes. +pub(crate) fn apply_column_statistics(stats: &mut Statistics, columns: Vec) { + if let Some(bytes) = columns.iter().try_fold(0_usize, |total, column| { + total.checked_add(*column.byte_size.get_value()?) + }) { + stats.total_byte_size = Precision::Inexact(bytes); + } + stats.column_statistics = columns; +} diff --git a/datafusion/distributed-iceberg/src/config.rs b/datafusion/distributed-iceberg/src/config.rs index 2f9e38c53..762088ff2 100644 --- a/datafusion/distributed-iceberg/src/config.rs +++ b/datafusion/distributed-iceberg/src/config.rs @@ -14,7 +14,8 @@ extensions_options! { pub row_group_filtering_enabled: bool, default = true /// Whether to apply row-level selections while reading Parquet files. pub row_selection_enabled: bool, default = false - /// Whether to include column statistics read during planning + /// Read column statistics during planning (retained from the files selected during planning). + /// Eager scans aggregate selected-file metrics and retain inexact snapshot NDV. pub column_stats_enabled: bool, default = false /// Discover and retain file tasks during physical planning instead of execution. pub eager_file_planning: bool, default = false diff --git a/datafusion/distributed-iceberg/src/distributed_desired_task_count_handler.rs b/datafusion/distributed-iceberg/src/distributed_desired_task_count_handler.rs index e4a15ebe4..7525e59ea 100644 --- a/datafusion/distributed-iceberg/src/distributed_desired_task_count_handler.rs +++ b/datafusion/distributed-iceberg/src/distributed_desired_task_count_handler.rs @@ -1,3 +1,4 @@ +use crate::common::scan_config::stats_from_snapshot; use std::num::NonZeroUsize; use datafusion::common::{Result, config_datafusion_err}; @@ -14,14 +15,21 @@ pub fn iceberg_desired_task_count( ev: DesiredTaskCountEvent, ) -> Option> { let node = ev.plan.downcast_ref::()?.data_source(); - if !node.is::() && !node.is::() { - return None; - } - let statistics = match node.partition_statistics(None) { - Ok(statistics) => statistics, - Err(error) => return Some(Err(error)), - }; - let total_bytes = *statistics.total_byte_size.get_value()?; + // Relation statistics may describe projected columns; parallelism uses whole-file work. + let (total_bytes, file_limit) = + if let Some(source) = node.downcast_ref::() { + let (count, bytes) = source.file_count_and_bytes()?; + (bytes, Some(count)) + } else { + let source = node.downcast_ref::()?; + source.feed.inner()?; + let statistics = + match stats_from_snapshot(source.table_snapshot.clone(), &source.scan.schema) { + Ok(statistics) => statistics, + Err(error) => return Some(Err(error)), + }; + (*statistics.total_byte_size.get_value()?, None) + }; let config = DistributedConfig::from_session_config(ev.session_config).ok()?; Some( @@ -31,11 +39,7 @@ pub fn iceberg_desired_task_count( ev.session_config.target_partitions(), ) .map(|count| { - let count = node - .downcast_ref::() - .map_or(count, |source| { - count.min(source.planned_files().count() as f64) - }); + let count = file_limit.map_or(count, |files| count.min(files as f64)); DesiredTaskCountEventResponse::soft(count) }), ) diff --git a/datafusion/distributed-iceberg/src/eager/data_source.rs b/datafusion/distributed-iceberg/src/eager/data_source.rs index 06df942f4..ae213ffda 100644 --- a/datafusion/distributed-iceberg/src/eager/data_source.rs +++ b/datafusion/distributed-iceberg/src/eager/data_source.rs @@ -90,6 +90,18 @@ impl EagerIcebergDataSource { .map(|index| &self.files.tasks()[index]) } + pub(crate) fn file_count_and_bytes(&self) -> Option<(usize, usize)> { + let mut count = 0; + let mut bytes = 0usize; + for group in self.partitions.iter() { + count += group.len(); + for &index in group { + bytes = bytes.checked_add(self.files.tasks()[index].length().try_into().ok()?)?; + } + } + Some((count, bytes)) + } + fn file_indices(&self) -> Vec { let mut indices: Vec<_> = self.partitions.iter().flatten().copied().collect(); indices.sort_unstable(); diff --git a/datafusion/distributed-iceberg/src/eager/planned_files.rs b/datafusion/distributed-iceberg/src/eager/planned_files.rs index eb7f7c9cf..d9f188b01 100644 --- a/datafusion/distributed-iceberg/src/eager/planned_files.rs +++ b/datafusion/distributed-iceberg/src/eager/planned_files.rs @@ -6,7 +6,8 @@ use datafusion::common::{Result, Statistics}; use datafusion::execution::TaskContext; use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation}; use futures::TryStreamExt; -use iceberg::scan::{FileScanTask, TableScan}; +use iceberg::scan::{FileScanTask, FileScanTaskColumnStats, TableScan}; +use iceberg::spec::{Datum, PrimitiveLiteral}; use crate::common::df_err; @@ -86,6 +87,9 @@ fn retained_size(task: &FileScanTask) -> usize { + size_of_val(task.project_field_ids()) + size_of_val(task.deletes()) + task.key_metadata().map_or(0, <[u8]>::len); + if let Some(stats) = task.column_stats() { + bytes = bytes.saturating_add(column_stats_size(stats)); + } for delete in task.deletes() { bytes = bytes .saturating_add(delete.file_path().len()) @@ -95,3 +99,29 @@ fn retained_size(task: &FileScanTask) -> usize { } bytes } + +/// Approximate heap size of the metrics a task retains for column statistics. +fn column_stats_size(stats: &FileScanTaskColumnStats) -> usize { + let counts = [ + stats.column_sizes(), + stats.value_counts(), + stats.null_value_counts(), + stats.nan_value_counts(), + ] + .iter() + .map(|counts| counts.len() * size_of::<(i32, u64)>()) + .sum::(); + let bounds = [stats.lower_bounds(), stats.upper_bounds()] + .iter() + .flat_map(|bounds| bounds.values()) + .map(|bound| { + size_of::<(i32, Datum)>() + + match bound.literal() { + PrimitiveLiteral::String(value) => value.len(), + PrimitiveLiteral::Binary(value) => value.len(), + _ => 0, + } + }) + .sum::(); + size_of::() + counts + bounds +} diff --git a/datafusion/distributed-iceberg/src/lazy/data_source.rs b/datafusion/distributed-iceberg/src/lazy/data_source.rs index 97339bab3..1f52ded6c 100644 --- a/datafusion/distributed-iceberg/src/lazy/data_source.rs +++ b/datafusion/distributed-iceberg/src/lazy/data_source.rs @@ -1,3 +1,4 @@ +use crate::common::scan_config::apply_column_statistics; use crate::common::scan_config::{ScanConfig, stats_from_snapshot}; use std::sync::Arc; @@ -180,7 +181,7 @@ impl DataSource for LazyIcebergDataSource { let mut stats = stats_from_snapshot(self.table_snapshot.clone(), &self.scan.schema)?; if let Some(col_stats) = &self.column_stats { - stats.column_statistics = col_stats.clone(); + apply_column_statistics(&mut stats, col_stats.clone()); } Ok(Arc::new(stats)) diff --git a/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs b/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs index 1328e63a7..fc5b97de1 100644 --- a/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs +++ b/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs @@ -1,17 +1,150 @@ use crate::common::df_err; use datafusion::common::runtime::JoinSet; use datafusion::common::stats::Precision; -use datafusion::common::{ColumnStatistics, HashMap, Result}; +use datafusion::common::{ColumnStatistics, HashMap, Result, exec_err}; use datafusion::error::DataFusionError; use datafusion::scalar::ScalarValue; use iceberg::puffin::APACHE_DATASKETCHES_THETA_V1; use iceberg::spec::{ - DataFile, Datum, ManifestContentType, ManifestList, PrimitiveLiteral, PrimitiveType, - SnapshotRef, + DataFile, Datum, ManifestContentType, ManifestList, NestedFieldRef, PrimitiveLiteral, + PrimitiveType, Schema, SnapshotRef, Type, }; use iceberg::table::Table; use std::sync::Arc; +use iceberg::scan::{FileScanTask, FileScanTaskColumnStats}; + +/// Names of the primitive leaf columns under `fields`, in schema-dotted form. Passed to +/// `TableScanBuilder::with_column_stats` so each planned `FileScanTask` carries the file's +/// metrics for exactly the columns these statistics cover. Leaves that are not primitive +/// (variants) have no file metrics and are skipped. +pub(crate) fn column_stats_names( + schema: &Schema, + fields: &[NestedFieldRef], +) -> Result> { + let mut names = vec![]; + for field in fields { + let mut leaves = vec![]; + leaves_of(field, &mut leaves); + for leaf in leaves { + if !matches!(leaf.field_type.as_ref(), Type::Primitive(_)) { + continue; + } + let Some(name) = schema.name_by_field_id(leaf.id) else { + return exec_err!("Iceberg column {} has no name in the scan schema", leaf.id); + }; + names.push(name.to_string()); + } + } + Ok(names) +} + +/// Selected-file metrics, with snapshot NDV retained only as an inexact fallback. +/// Metrics come from the `FileScanTask::column_stats` that the scan retained while +/// planning (see [`column_stats_names`]), so no manifest is read a second time. +pub(crate) fn selected_column_statistics( + table: &Table, + snapshot: &SnapshotRef, + fields: &[NestedFieldRef], + files: &[FileScanTask], + may_filter_rows: bool, +) -> Result> { + if files.is_empty() || fields.is_empty() { + return Ok(vec![ + ColumnStatistics { + null_count: Precision::Exact(0), + distinct_count: Precision::Exact(0), + byte_size: Precision::Inexact(0), + ..ColumnStatistics::new_unknown() + }; + fields.len() + ]); + } + if files + .iter() + .any(|task| task.start() != 0 || task.length() != task.file_size_in_bytes()) + { + // Full-file metrics cannot describe a byte-range task. + return Ok(vec![ColumnStatistics::new_unknown(); fields.len()]); + } + let field_ids: Vec<_> = fields + .iter() + .map(|field| { + let mut ids = vec![]; + leaf_ids(field, &mut ids); + ids + }) + .collect(); + let mut statistics = vec![None; fields.len()]; + for task in files { + let Some(file) = task.column_stats() else { + return exec_err!( + "Planned Iceberg file {} carries no column statistics", + task.data_file_path() + ); + }; + for ((acc, field), ids) in statistics.iter_mut().zip(fields).zip(&field_ids) { + // Leaf metrics cannot describe a parent struct/list/map's value or null count. + let mut next = if matches!(field.field_type.as_ref(), Type::Primitive(_)) { + task_col_stats(file, field.id) + } else { + ColumnStatistics::new_unknown() + }; + next.byte_size = ids + .iter() + .try_fold(0_usize, |total, id| { + let size = usize::try_from(*file.column_sizes().get(id)?).ok()?; + total.checked_add(size) + }) + .map(Precision::Inexact) + .unwrap_or(Precision::Absent); + *acc = Some(merge_col_stats(acc.take(), next)); + } + } + let rows = files.iter().try_fold(0_usize, |rows, task| { + rows.checked_add(usize::try_from(task.record_count()?).ok()?) + }); + let ndvs = ndv_from_metadata(table, snapshot.snapshot_id()); + Ok(statistics + .into_iter() + .zip(fields) + .map(|(stats, field)| { + let mut stats = stats.unwrap_or_else(ColumnStatistics::new_unknown); + if may_filter_rows { + stats.null_count = stats.null_count.to_inexact(); + } + // Snapshot sketches cannot be restricted to selected files. Retain their + // estimate, capped by the selected rows, without claiming exactness. + if let Some(&ndv) = ndvs.get(&field.id) { + stats.distinct_count = Precision::Inexact(rows.map_or(ndv, |rows| ndv.min(rows))); + } + stats + }) + .collect()) +} + +fn leaf_ids(field: &NestedFieldRef, ids: &mut Vec) { + let mut leaves = vec![]; + leaves_of(field, &mut leaves); + ids.extend(leaves.iter().map(|leaf| leaf.id)); +} + +fn leaves_of<'a>(field: &'a NestedFieldRef, leaves: &mut Vec<&'a NestedFieldRef>) { + match field.field_type.as_ref() { + Type::Primitive(_) | Type::Variant(_) => leaves.push(field), + Type::Struct(fields) => { + for field in fields.fields() { + leaves_of(field, leaves); + } + } + Type::List(list) => leaves_of(&list.element_field, leaves), + Type::Map(map) => { + leaves_of(&map.key_field, leaves); + leaves_of(&map.value_field, leaves); + } + } +} + /// Getting number of distinct values for columns in a table /// /// Note: https://iceberg.apache.org/puffin-spec/#apache-datasketches-theta-v1-blob-type @@ -139,7 +272,13 @@ fn merge_col_stats(acc: Option, next: ColumnStatistics) -> Col null_count: acc.null_count.add(&next.null_count), min_value: acc.min_value.min(&next.min_value), max_value: acc.max_value.max(&next.max_value), - byte_size: acc.byte_size.add(&next.byte_size), + byte_size: acc + .byte_size + .get_value() + .zip(next.byte_size.get_value()) + .and_then(|(a, b)| a.checked_add(*b)) + .map(Precision::Inexact) + .unwrap_or(Precision::Absent), sum_value: Precision::Absent, distinct_count: Precision::Absent, }, @@ -147,28 +286,43 @@ fn merge_col_stats(acc: Option, next: ColumnStatistics) -> Col } fn data_file_col_stats(df: &DataFile, id: i32) -> ColumnStatistics { + file_col_stats( + df.null_value_counts().get(&id).copied(), + df.lower_bounds().get(&id), + df.upper_bounds().get(&id), + df.column_sizes().get(&id).copied(), + ) +} + +fn task_col_stats(stats: &FileScanTaskColumnStats, id: i32) -> ColumnStatistics { + file_col_stats( + stats.null_value_counts().get(&id).copied(), + stats.lower_bounds().get(&id), + stats.upper_bounds().get(&id), + stats.column_sizes().get(&id).copied(), + ) +} + +fn file_col_stats( + null_count: Option, + lower_bound: Option<&Datum>, + upper_bound: Option<&Datum>, + column_size: Option, +) -> ColumnStatistics { ColumnStatistics { - null_count: df - .null_value_counts() - .get(&id) - .map(|n| Precision::Exact(*n as usize)) + null_count: null_count + .map(|n| Precision::Exact(n as usize)) .unwrap_or(Precision::Absent), - min_value: df - .lower_bounds() - .get(&id) + min_value: lower_bound .and_then(datum_to_scalar) .map(Precision::Inexact) .unwrap_or(Precision::Absent), - max_value: df - .upper_bounds() - .get(&id) + max_value: upper_bound .and_then(datum_to_scalar) .map(Precision::Inexact) .unwrap_or(Precision::Absent), - byte_size: df - .column_sizes() - .get(&id) - .map(|n| Precision::Inexact(*n as usize)) + byte_size: column_size + .map(|n| Precision::Inexact(n as usize)) .unwrap_or(Precision::Absent), sum_value: Precision::Absent, distinct_count: Precision::Absent, diff --git a/datafusion/distributed-iceberg/src/table_provider/scan.rs b/datafusion/distributed-iceberg/src/table_provider/scan.rs index aeeb2ec94..943332c77 100644 --- a/datafusion/distributed-iceberg/src/table_provider/scan.rs +++ b/datafusion/distributed-iceberg/src/table_provider/scan.rs @@ -1,8 +1,10 @@ use crate::common::df_err; -use crate::common::scan_config::ScanConfig; +use crate::common::scan_config::{ScanConfig, apply_column_statistics}; use crate::eager::assign_files; use crate::eager::planned_files::PlannedFiles; -use crate::table_provider::column_statistics::compute_column_stats; +use crate::table_provider::column_statistics::{ + column_stats_names, compute_column_stats, selected_column_statistics, +}; use crate::{EagerIcebergDataSource, IcebergConfig, IcebergWorkUnitFeed, LazyIcebergDataSource}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{Result, config_err, exec_err}; @@ -60,6 +62,25 @@ pub(crate) async fn plan_scan( .unwrap_or_else(iceberg::Runtime::current), &context, ); + // The columns whose statistics are requested, with the schema that names them. + let stats_fields = if IcebergConfig::from_task_context(&context).column_stats_enabled { + let schema = match &table_snapshot { + Some(snap) => snap.schema(table.metadata()).map_err(df_err)?, + // empty table + None => table.metadata().current_schema().clone(), + }; + let fields = schema.as_struct().fields(); + let fields: Vec<_> = match opts.projection { + Some(projection) => projection + .iter() + .map(|&idx| Arc::clone(&fields[idx])) + .collect(), + None => fields.to_vec(), + }; + Some((schema, fields)) + } else { + None + }; let planned = if IcebergConfig::from_task_context(&context).eager_file_planning { if partitioning.partition_count() == 0 { return config_err!("Iceberg scan requires at least one output partition"); @@ -74,6 +95,10 @@ pub(crate) async fn plan_scan( if let Some(predicates) = &predicates { builder = builder.with_filter(predicates.clone()); } + if let Some((schema, fields)) = &stats_fields { + // Keep each file's metrics on its task, so statistics need no second manifest pass. + builder = builder.with_column_stats(column_stats_names(schema, fields)?); + } let table_scan = builder.build().map_err(df_err)?; // File discovery runs on Iceberg's I/O runtime and is canceled if planning is dropped. let (_cancel_on_drop, canceled) = oneshot::channel::<()>(); @@ -84,7 +109,7 @@ pub(crate) async fn plan_scan( planned = PlannedFiles::collect(table_scan, &planning_context) => planned, } }); - Some(Arc::new(task.await.map_err(df_err)??)) + Some(task.await.map_err(df_err)??) } else { None }; @@ -92,27 +117,29 @@ pub(crate) async fn plan_scan( || planned .as_ref() .is_some_and(|files| files.tasks().iter().any(|task| !task.deletes().is_empty())); - let column_stats = if IcebergConfig::from_task_context(&context).column_stats_enabled - && !(planned.is_some() && may_filter_rows) - { - let schema = match &table_snapshot { - Some(snap) => snap.schema(table.metadata()).map_err(df_err)?, - // empty table - None => table.metadata().current_schema().clone(), - }; - let fields = schema.as_struct().fields().to_vec(); - let field_ids: Vec = match opts.projection { - Some(projection) => projection.iter().map(|&idx| fields[idx].id).collect(), - None => fields.iter().map(|f| f.id).collect(), - }; - Some(compute_column_stats(table.clone(), field_ids, table_snapshot.clone()).await?) + let column_stats = if let Some((_, fields)) = &stats_fields { + if let Some(files) = &planned + && let Some(snapshot) = &table_snapshot + { + Some(selected_column_statistics( + &table, + snapshot, + fields, + files.tasks(), + may_filter_rows, + )?) + } else { + let field_ids = fields.iter().map(|field| field.id).collect(); + Some(compute_column_stats(table.clone(), field_ids, table_snapshot.clone()).await?) + } } else { None }; if let Some(planned) = planned { + let planned = Arc::new(planned); let mut statistics = planned.statistics(&scan.schema, may_filter_rows); if let Some(column_stats) = column_stats { - statistics.column_statistics = column_stats; + apply_column_statistics(&mut statistics, column_stats); } let partitions = assign_files(planned.tasks().iter(), 1, partitioning.partition_count())? .pop() diff --git a/datafusion/distributed-iceberg/tests/desired_task_count.rs b/datafusion/distributed-iceberg/tests/desired_task_count.rs index 90f792a31..99e9fa6f8 100644 --- a/datafusion/distributed-iceberg/tests/desired_task_count.rs +++ b/datafusion/distributed-iceberg/tests/desired_task_count.rs @@ -199,6 +199,8 @@ mod tests { let mut builder = IcebergTestHarness::builder() .with_session_extension(|state| state.with_iceberg_integration(Default::default())) .with_table(create_taxi_table().await?) + // Column statistics must not change how much work the estimator sees. + .with_config_option("iceberg.column_stats_enabled", true) .configure_session(|mut state| { let config = state.config().get_or_insert_default(); config.options_mut().execution.target_partitions = 2; diff --git a/datafusion/distributed-iceberg/tests/nested_column_statistics.rs b/datafusion/distributed-iceberg/tests/nested_column_statistics.rs new file mode 100644 index 000000000..39c78d9b6 --- /dev/null +++ b/datafusion/distributed-iceberg/tests/nested_column_statistics.rs @@ -0,0 +1,132 @@ +#[cfg(test)] +mod tests { + use std::error::Error; + use std::sync::Arc; + + use datafusion::common::stats::Precision; + use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_plan::statistics::{StatisticsArgs, StatisticsContext}; + use datafusion_distributed_iceberg::IcebergExt; + use datafusion_distributed_iceberg_test_tools::{FixtureCatalog, IcebergTestHarness}; + use iceberg::TableIdent; + use iceberg::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, NestedField, + PrimitiveType, Schema, Struct, StructType, Type, + }; + + #[tokio::test] + async fn selected_nested_columns_sum_leaves_and_keep_missing_leaves_unknown() + -> Result<(), Box> { + let harness = nested_fixture().await?; + for (id, expected) in [(1, Precision::Inexact(300)), (2, Precision::Absent)] { + let plan = harness + .physical_plan(&format!("SELECT details FROM nested WHERE id = {id}")) + .await?; + let mut checked = false; + plan.apply(|node| { + if node.is::() { + let stats = + StatisticsContext::new().compute(node.as_ref(), &StatisticsArgs::new())?; + let column = node.schema().index_of("details")?; + assert_eq!(stats.column_statistics[column].byte_size, expected); + assert_eq!( + stats.column_statistics[column].null_count, + Precision::Absent + ); + assert_eq!(stats.column_statistics[column].min_value, Precision::Absent); + assert_eq!(stats.column_statistics[column].max_value, Precision::Absent); + assert_eq!(stats.num_rows, Precision::Inexact(10)); + checked = true; + } + Ok(TreeNodeRecursion::Continue) + })?; + assert!(checked); + } + Ok(()) + } + + // A table of two synthetic whole files that are never opened. The first sizes every + // leaf of `details`; the second has no size for the list element, so the struct's + // size cannot be reconstructed from its leaves. + async fn nested_fixture() -> Result> { + let schema = Schema::builder() + .with_fields([ + Arc::new(NestedField::required( + 1, + "id", + Type::Primitive(PrimitiveType::Int), + )), + Arc::new(NestedField::optional( + 2, + "details", + Type::Struct(StructType::new(vec![ + Arc::new(NestedField::optional( + 3, + "name", + Type::Primitive(PrimitiveType::String), + )), + Arc::new(NestedField::optional( + 4, + "values", + Type::List(ListType::new(Arc::new(NestedField::optional( + 5, + "element", + Type::Primitive(PrimitiveType::Long), + )))), + )), + ])), + )), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let table = fixtures + .table(TableIdent::from_strs(["default", "nested"])?, schema) + .create() + .await?; + // Field ids as the catalog assigned them when creating the table. + let schema = Arc::clone(table.table().metadata().current_schema()); + let field_id = |name: &str| { + schema + .field_id_by_name(name) + .ok_or_else(|| format!("no field {name}")) + }; + let (id, name, element) = ( + field_id("id")?, + field_id("details.name")?, + field_id("details.values.element")?, + ); + let location = table.table().metadata().location().to_owned(); + let mut files = Vec::new(); + for file in [1, 2] { + let sizes = if file == 1 { + [(id, 40), (name, 100), (element, 200)] + .into_iter() + .collect() + } else { + [(id, 40), (name, 900)].into_iter().collect() + }; + files.push( + DataFileBuilder::default() + .content(DataContentType::Data) + .file_format(DataFileFormat::Parquet) + .file_path(format!("{location}/data/unopened-nested-{file}.parquet")) + .partition(Struct::empty()) + .record_count(10) + .file_size_in_bytes(1000) + .column_sizes(sizes) + .lower_bounds([(id, Datum::int(file))].into()) + .upper_bounds([(id, Datum::int(file))].into()) + .build()?, + ); + } + Ok(IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(table) + .with_data_files(files) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.column_stats_enabled", true) + .build() + .await?) + } +} diff --git a/datafusion/distributed-iceberg/tests/statistics.rs b/datafusion/distributed-iceberg/tests/statistics.rs index e7b711412..6b7241fd5 100644 --- a/datafusion/distributed-iceberg/tests/statistics.rs +++ b/datafusion/distributed-iceberg/tests/statistics.rs @@ -13,18 +13,27 @@ mod tests { use datafusion::arrow::array::record_batch; // `record_batch!` refers to `arrow_schema` by name. use datafusion::arrow::datatypes as arrow_schema; + use datafusion::common::internal_datafusion_err; use datafusion::common::stats::Precision; + use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion::common::{ColumnStatistics, Statistics}; + use datafusion::datasource::source::DataSourceExec; use datafusion::error::Result; use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties}; use datafusion::parquet::schema::types::ColumnPath; + use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::displayable; use datafusion::physical_plan::statistics::{StatisticsArgs, StatisticsContext}; use datafusion::scalar::ScalarValue; + use datafusion_distributed::{DesiredTaskCountEvent, DistributedExt}; + use datafusion_distributed_iceberg::iceberg_desired_task_count; + use datafusion_distributed_iceberg_test_tools::IcebergTestHarnessBuilder; use iceberg::TableIdent; + use iceberg::puffin::APACHE_DATASKETCHES_THETA_V1; use iceberg::spec::{ - DataFile, MAIN_BRANCH, NestedField, Operation, PrimitiveType, Schema, Snapshot, Summary, - TableMetadata, Type, + BlobMetadata, DataContentType, DataFile, DataFileBuilder, DataFileFormat, Datum, Literal, + MAIN_BRANCH, NestedField, Operation, PrimitiveType, Schema, Snapshot, StatisticsFile, + Struct, Summary, TableMetadata, Type, }; // Values from the generated taxi snapshot summary. @@ -142,13 +151,14 @@ mod tests { let stats = query_statistics(&harness, "SELECT * FROM metrics").await?; assert_eq!(stats.num_rows, Precision::Exact(METRICS_ROWS)); + // With a size for every column, summed column bytes replace whole-file bytes. assert_eq!( stats.total_byte_size, - Precision::Exact( + Precision::Inexact( files .iter() - .map(|file| file.file_size_in_bytes() as usize) - .sum() + .flat_map(|file| file.column_sizes().values()) + .sum::() as usize ) ); assert_eq!( @@ -456,6 +466,269 @@ mod tests { Ok((table, files)) } + // ---- Selected-file (eager) column statistics over synthetic manifest metrics ---- + // + // These fixtures describe files through manifest entries alone; the data paths are + // never opened. Metric keys are field ids: 1 is vendor_id and 4 is passenger_count, + // not schema indexes. trip_distance (id 5) has no metrics. The snapshot keeps the + // generated taxi summary, so summary totals disagree with the synthetic files and + // selected-file metrics are distinguishable. + + // Each synthetic file's record count and size; two files replace the taxi manifest. + const SYNTHETIC_ROWS: usize = 87_500; + const SYNTHETIC_BYTES: usize = 2_240_191; + + #[tokio::test] + async fn eager_full_scan_merges_selected_file_metrics() -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let stats = query_statistics(&harness, "SELECT * FROM taxi").await?; + assert_eq!(stats.num_rows, Precision::Exact(2 * SYNTHETIC_ROWS)); + assert_eq!(stats.total_byte_size, Precision::Exact(2 * SYNTHETIC_BYTES)); + let mut columns = vec![ColumnStatistics::new_unknown(); 13]; + columns[0] = expected_vendor_id_statistics(); + columns[3] = expected_passenger_count_statistics(); + assert_eq!(stats.column_statistics, columns); + Ok(()) + } + + #[tokio::test] + async fn eager_projection_reorders_selected_file_metrics() -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let stats = query_statistics( + &harness, + "SELECT passenger_count, vendor_id, trip_distance FROM taxi", + ) + .await?; + assert_eq!(stats.num_rows, Precision::Exact(2 * SYNTHETIC_ROWS)); + assert_eq!( + stats.column_statistics, + vec![ + expected_passenger_count_statistics(), + expected_vendor_id_statistics(), + ColumnStatistics::new_unknown(), + ] + ); + Ok(()) + } + + #[tokio::test] + async fn filtered_statistics_use_selected_files_and_cap_snapshot_ndv() + -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + // Manifest bounds select only the first file: passenger_count spans 10 to 20. + let source = scan_node( + &harness + .physical_plan("SELECT vendor_id FROM taxi WHERE passenger_count < 25") + .await?, + )?; + let stats = StatisticsContext::new().compute(source.as_ref(), &StatisticsArgs::new())?; + let column = source.schema().index_of("vendor_id")?; + assert_eq!(stats.num_rows, Precision::Inexact(SYNTHETIC_ROWS)); + // First file: 100 bytes. A retained-row fraction would incorrectly yield 200. + assert_eq!( + stats.column_statistics[column].byte_size, + Precision::Inexact(100) + ); + assert_eq!( + stats.column_statistics[column].min_value, + Precision::Inexact(ScalarValue::Int32(Some(2))) + ); + assert_eq!( + stats.column_statistics[column].max_value, + Precision::Inexact(ScalarValue::Int32(Some(9))) + ); + // The snapshot sketch claims 100,000 distinct values; the selected rows cap it. + assert_eq!( + stats.column_statistics[column].distinct_count, + Precision::Inexact(SYNTHETIC_ROWS) + ); + assert_eq!( + stats.column_statistics[column].null_count, + Precision::Inexact(2) + ); + // Both scan columns have sizes: vendor_id 100 + passenger_count 200. + assert_eq!(stats.total_byte_size, Precision::Inexact(300)); + Ok(()) + } + + #[tokio::test] + async fn incomplete_filtered_metrics_keep_unknown_columns_and_file_byte_fallback() + -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let source = scan_node( + &harness + .physical_plan("SELECT trip_distance FROM taxi WHERE passenger_count < 25") + .await?, + )?; + let stats = StatisticsContext::new().compute(source.as_ref(), &StatisticsArgs::new())?; + let column = source.schema().index_of("trip_distance")?; + assert_eq!(stats.column_statistics[column].byte_size, Precision::Absent); + // An unknown column size keeps the selected file's total bytes instead. + assert_eq!(stats.total_byte_size, Precision::Inexact(SYNTHETIC_BYTES)); + Ok(()) + } + + #[tokio::test] + async fn filtered_scan_merges_bounds_and_preserves_missing_null_counts() + -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let source = scan_node( + &harness + .physical_plan( + "SELECT passenger_count, vendor_id FROM taxi WHERE passenger_count < 100", + ) + .await?, + )?; + let stats = StatisticsContext::new().compute(source.as_ref(), &StatisticsArgs::new())?; + let passenger = &stats.column_statistics[source.schema().index_of("passenger_count")?]; + assert_eq!( + passenger.min_value, + Precision::Inexact(ScalarValue::Int64(Some(10))) + ); + assert_eq!( + passenger.max_value, + Precision::Inexact(ScalarValue::Int64(Some(40))) + ); + // One file omits this column's null count: the total must stay unknown. + assert_eq!(passenger.null_count, Precision::Absent); + assert_eq!(passenger.distinct_count, Precision::Absent); + let vendor = &stats.column_statistics[source.schema().index_of("vendor_id")?]; + assert_eq!(vendor.null_count, Precision::Inexact(5)); + assert_eq!(vendor.distinct_count, Precision::Inexact(100_000)); + Ok(()) + } + + #[tokio::test] + async fn projected_bytes_do_not_reduce_scan_task_count() -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let source = scan_node( + &harness + .physical_plan("SELECT vendor_id FROM taxi WHERE passenger_count < 100") + .await?, + )?; + let stats = StatisticsContext::new().compute(source.as_ref(), &StatisticsArgs::new())?; + assert_eq!(stats.total_byte_size, Precision::Inexact(1000)); + // 4.48 MB of file work / 1 MB / 1 partition, capped at the two files. + let estimate = iceberg_desired_task_count(DesiredTaskCountEvent { + plan: &source, + session_config: harness.state().config(), + }) + .transpose()? + .map(|response| response.task_count.as_usize()); + assert_eq!(estimate, Some(2)); + Ok(()) + } + + fn scan_node(plan: &Arc) -> Result> { + let mut source = None; + plan.apply(|node| { + if node.is::() { + source = Some(Arc::clone(node)); + return Ok(TreeNodeRecursion::Stop); + } + Ok(TreeNodeRecursion::Continue) + })?; + source.ok_or_else(|| internal_datafusion_err!("expected an Iceberg scan")) + } + + // Planning-only eager fixture over the taxi table's current snapshot. + async fn harness_with_manifest_metrics() -> Result> { + Ok(manifest_metrics_builder() + .await? + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.column_stats_enabled", true) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + state.with_distributed_file_scan_config_bytes_per_partition(1_000_000) + })? + .build() + .await?) + } + + async fn manifest_metrics_builder() -> Result> { + let table = create_taxi_table().await?; + let location = table.table().metadata().location().to_owned(); + let mut file = DataFileBuilder::default(); + file.content(DataContentType::Data) + .file_format(DataFileFormat::Parquet) + .record_count(SYNTHETIC_ROWS as u64) + .file_size_in_bytes(SYNTHETIC_BYTES as u64) + .partition(Struct::from_iter([Some(Literal::date_from_str( + "2024-01-10", + )?)])); + let first = file + .file_path(format!("{location}/data/unopened-metrics-first.parquet")) + .null_value_counts([(1, 2), (4, 4)].into()) + .column_sizes([(1, 100), (4, 200)].into()) + .lower_bounds([(1, Datum::int(2)), (4, Datum::long(10))].into()) + .upper_bounds([(1, Datum::int(9)), (4, Datum::long(20))].into()) + .build()?; + let second = file + .file_path(format!("{location}/data/unopened-metrics-second.parquet")) + .null_value_counts([(1, 3)].into()) + .column_sizes([(1, 300), (4, 400)].into()) + .lower_bounds([(1, Datum::int(1)), (4, Datum::long(30))].into()) + .upper_bounds([(1, Datum::int(5)), (4, Datum::long(40))].into()) + .build()?; + Ok(IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(table) + .with_metadata(vendor_id_ndv_sketch) + .with_data_files([first, second])) + } + + // Attaches a snapshot statistics file claiming 100,000 distinct vendor ids. + fn vendor_id_ndv_sketch(metadata: &TableMetadata) -> TableMetadata { + let snapshot = metadata.current_snapshot().expect("taxi has a snapshot"); + let statistics = StatisticsFile { + snapshot_id: snapshot.snapshot_id(), + statistics_path: format!( + "{}/metadata/unopened-statistics.puffin", + metadata.location() + ), + file_size_in_bytes: 0, + file_footer_size_in_bytes: 0, + key_metadata: None, + blob_metadata: vec![BlobMetadata { + r#type: APACHE_DATASKETCHES_THETA_V1.to_owned(), + snapshot_id: snapshot.snapshot_id(), + sequence_number: snapshot.sequence_number(), + fields: vec![1], + properties: [("ndv".to_owned(), "100000".to_owned())].into(), + }], + }; + metadata + .clone() + .into_builder(None) + .set_statistics(statistics) + .build() + .expect("taxi metadata stays valid") + .metadata + } + + // The merged vendor_id metrics of both synthetic files, including the capped sketch. + fn expected_vendor_id_statistics() -> ColumnStatistics { + ColumnStatistics { + null_count: Precision::Exact(5), + distinct_count: Precision::Inexact(100_000), + min_value: Precision::Inexact(ScalarValue::Int32(Some(1))), + max_value: Precision::Inexact(ScalarValue::Int32(Some(9))), + byte_size: Precision::Inexact(400), + ..ColumnStatistics::new_unknown() + } + } + + fn expected_passenger_count_statistics() -> ColumnStatistics { + ColumnStatistics { + min_value: Precision::Inexact(ScalarValue::Int64(Some(10))), + max_value: Precision::Inexact(ScalarValue::Int64(Some(40))), + byte_size: Precision::Inexact(600), + // One file omits this column's null count: the total must stay unknown. + ..ColumnStatistics::new_unknown() + } + } + // The column statistics of the metrics table, in schema order, with column stats enabled. fn expected_column_statistics(files: &[DataFile]) -> [ColumnStatistics; 3] { let size = |field_id| {