From fdb7704108c97cdfa4b58ff34d60df3cd7c1caf8 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Tue, 20 Oct 2026 10:29:50 -0400 Subject: [PATCH 1/4] feat: reconstruct eager Iceberg column statistics from selected files (cherry picked from commit 692e362acca99f6c8e5ed2f377608758ccc09d31) (cherry picked from commit 0a4193b676ad02c1de11011f2a34d2420e54e7a2) (cherry picked from commit 4edf0ec794c7287bfcf4e136e883aedb312a2e13) --- .../src/harness.rs | 116 ++++++- datafusion/distributed-iceberg/README.md | 22 +- .../src/common/scan_config.rs | 10 + datafusion/distributed-iceberg/src/config.rs | 3 +- .../distributed_desired_task_count_handler.rs | 19 +- .../src/lazy/data_source.rs | 3 +- .../src/table_provider/column_statistics.rs | 140 ++++++++- .../src/table_provider/scan.rs | 39 ++- .../tests/desired_task_count.rs | 2 + .../tests/nested_column_statistics.rs | 132 ++++++++ .../distributed-iceberg/tests/statistics.rs | 283 +++++++++++++++++- 11 files changed, 729 insertions(+), 40 deletions(-) create mode 100644 datafusion/distributed-iceberg/tests/nested_column_statistics.rs diff --git a/datafusion/distributed-iceberg-test-tools/src/harness.rs b/datafusion/distributed-iceberg-test-tools/src/harness.rs index 57eee91c6..7b762534d 100644 --- a/datafusion/distributed-iceberg-test-tools/src/harness.rs +++ b/datafusion/distributed-iceberg-test-tools/src/harness.rs @@ -20,7 +20,10 @@ use datafusion_distributed::{ use datafusion_proto::physical_plan::AsExecutionPlan; use datafusion_proto::protobuf::PhysicalPlanNode; use iceberg::MetadataLocation; -use iceberg::spec::{MAIN_BRANCH, TableMetadata, TableMetadataBuilder}; +use iceberg::spec::{ + DataFile, MAIN_BRANCH, ManifestListWriter, ManifestWriterBuilder, Operation, Snapshot, Summary, + TableMetadata, TableMetadataBuilder, +}; use crate::FixtureTable; use crate::error::df_err; @@ -117,6 +120,7 @@ pub struct IcebergTestHarnessBuilder { table_options: BTreeMap, table: Option, metadata: Option, + data_files: Option>, workers: Option, } @@ -132,6 +136,7 @@ impl Default for IcebergTestHarnessBuilder { table_options: BTreeMap::new(), table: None, metadata: None, + data_files: None, workers: None, } } @@ -191,6 +196,16 @@ impl IcebergTestHarnessBuilder { self } + /// Replaces the current snapshot's manifest with synthetic file entries, creating a + /// snapshot if needed. Data files are not written and the snapshot's summary totals are + /// preserved, so tests can tell summary statistics from per-file manifest metrics. + /// The entries' paths are never opened, so planning-only tests can describe files, + /// metrics and bounds directly. It applies after [`Self::with_metadata`]'s transform. + pub fn with_data_files(mut self, files: impl IntoIterator) -> Self { + self.data_files = Some(files.into_iter().collect()); + self + } + /// Enables distributed planning with logical workers backed by in-memory gRPC. pub fn with_workers(mut self, workers: usize) -> Self { self.workers = Some(workers); @@ -210,24 +225,34 @@ impl IcebergTestHarnessBuilder { harness.query_with_metrics(sql).await } - /// Registers the table, if any, from customized metadata if [`Self::with_metadata`] was - /// given. + /// Registers the table, if any, from customized metadata if [`Self::with_metadata`] or + /// [`Self::with_data_files`] was given. pub async fn build(mut self) -> Result { - let table = match (self.table.take(), self.metadata.take()) { - (None, None) => None, - (None, Some(_)) => { + let table = match ( + self.table.take(), + self.metadata.take(), + self.data_files.take(), + ) { + (None, None, None) => None, + (None, _, _) => { return exec_err!("table metadata can only be customized for a table"); } - (Some(table), None) => { + (Some(table), None, None) => { let location = table.metadata_location()?.to_string(); Some((table, location)) } - (Some(table), Some(metadata)) => { - let metadata = metadata(table.table().metadata()); + (Some(table), metadata, data_files) => { + let mut customized = match metadata { + Some(metadata) => metadata(table.table().metadata()), + None => table.table().metadata().clone(), + }; + if let Some(files) = data_files { + customized = replace_manifest_with(&table, customized, files).await?; + } let location = MetadataLocation::from_str(table.metadata_location()?) .map_err(df_err)? .with_next_version(); - metadata + customized .write_to(table.table().file_io(), &location) .await .map_err(df_err)?; @@ -297,6 +322,77 @@ pub fn data_file_name(path: &str) -> &str { path.rsplit_once("/data/").map_or(path, |(_, name)| name) } +/// Writes a manifest of `files` and points the current snapshot's manifest list at it, +/// creating a snapshot when the metadata has none. Snapshot identity and summary are kept. +async fn replace_manifest_with( + table: &FixtureTable, + metadata: TableMetadata, + files: Vec, +) -> Result { + let metadata = if metadata.current_snapshot().is_none() { + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_sequence_number(1) + .with_timestamp_ms(metadata.last_updated_ms() + 1) + .with_manifest_list(format!( + "{}/metadata/synthetic-manifest-list.avro", + metadata.location() + )) + .with_schema_id(metadata.current_schema_id()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: Default::default(), + }) + .build(); + metadata + .into_builder(None) + .set_branch_snapshot(snapshot, MAIN_BRANCH) + .map_err(df_err)? + .build() + .map_err(df_err)? + .metadata + } else { + metadata + }; + let snapshot = metadata + .current_snapshot() + .expect("snapshot initialized above"); + let file_io = table.table().file_io(); + let manifest_path = format!("{}/metadata/synthetic-manifest.avro", metadata.location()); + for path in [manifest_path.as_str(), snapshot.manifest_list()] { + if file_io.exists(path).await.map_err(df_err)? { + file_io.delete(path).await.map_err(df_err)?; + } + } + let mut writer = ManifestWriterBuilder::new( + file_io.new_output(&manifest_path).map_err(df_err)?, + Some(snapshot.snapshot_id()), + Arc::clone(metadata.current_schema()), + metadata.default_partition_spec().as_ref().clone(), + ) + .build_v2_data(); + for file in files { + writer + .add_file(file, snapshot.sequence_number()) + .map_err(df_err)?; + } + let manifest = writer.write_manifest_file().await.map_err(df_err)?; + let mut list = ManifestListWriter::v2( + file_io + .new_output(snapshot.manifest_list()) + .map_err(df_err)? + .writer() + .await + .map_err(df_err)?, + snapshot.snapshot_id(), + snapshot.parent_snapshot_id(), + snapshot.sequence_number(), + ); + list.add_manifests([manifest].into_iter()).map_err(df_err)?; + list.close().await.map_err(df_err)?; + Ok(metadata) +} + /// Starts from `metadata` with its schema, partition spec and properties, but no snapshots. pub fn without_snapshots(metadata: &TableMetadata) -> TableMetadataBuilder { let snapshot_ids: Vec<_> = metadata.snapshots().map(|s| s.snapshot_id()).collect(); diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index 9c5b6507b..ce512f9dd 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -84,10 +84,9 @@ reading and deliver descriptors through the coordinator's feed. Eager scans derive row counts and bytes from selected files and cap task estimates by the number of selected files. Predicates or deletes make row counts inexact; -empty selections have exactly zero rows. Whole-snapshot column statistics are -skipped when they would describe a different row set. Lazy scans retain snapshot -statistics and task-size estimates. Selected-file column statistics remain a -separate change. Task-local eager variants and decoded plans report unknown +empty selections have exactly zero rows. When column statistics are enabled, +eager scans aggregate selected-file metrics. Lazy scans retain snapshot statistics +and task-size estimates. Task-local eager variants and decoded plans report unknown statistics rather than applying whole-snapshot aggregates to a subset. Both scans keep the filters DataFusion pushes into them, including the dynamic @@ -98,6 +97,21 @@ filter. Enable `iceberg.dynamic_filtering_enabled` to make Iceberg scans consume dynamic filters pushed into them. With it on, `EXPLAIN` shows them on the scan as `dynamic_filters=`. Scans don't skip data with these filters yet. +## Selected-file statistics + +With `iceberg.column_stats_enabled=true`, eager scans reconstruct column bytes, +min/max bounds, and null counts from the **selected files**. Missing metrics remain +unknown. Bounds are inexact; null counts also become inexact when predicates or +deletes may remove rows. Nested column bytes sum leaf sizes without inventing +parent bounds or null counts. + +Snapshot distinct-count estimates are retained as an inexact fallback, capped by +selected-file row counts when known. They are not recomputed for the selected +files and can overestimate the filtered result. Sum statistics remain unknown. + +Another manifest pass is needed because the pinned Iceberg API does not retain +column metrics in file tasks. Scan task counts continue to use whole-file bytes. + ```bash cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools ``` diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 71eebfe19..527f6665a 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -244,3 +244,13 @@ pub(crate) fn stats_from_snapshot( column_statistics: vec![ColumnStatistics::new_unknown(); schema.fields().len()], }) } + +/// Use projected compressed bytes when all column sizes are known, otherwise retain file bytes. +pub(crate) fn apply_column_statistics(stats: &mut Statistics, columns: Vec) { + if let Some(bytes) = columns.iter().try_fold(0_usize, |total, column| { + total.checked_add(*column.byte_size.get_value()?) + }) { + stats.total_byte_size = Precision::Inexact(bytes); + } + stats.column_statistics = columns; +} diff --git a/datafusion/distributed-iceberg/src/config.rs b/datafusion/distributed-iceberg/src/config.rs index 2f9e38c53..35aac75fa 100644 --- a/datafusion/distributed-iceberg/src/config.rs +++ b/datafusion/distributed-iceberg/src/config.rs @@ -14,7 +14,8 @@ extensions_options! { pub row_group_filtering_enabled: bool, default = true /// Whether to apply row-level selections while reading Parquet files. pub row_selection_enabled: bool, default = false - /// Whether to include column statistics read during planning + /// Read column statistics during planning (may require another manifest pass). + /// Eager scans aggregate selected-file metrics and retain inexact snapshot NDV. pub column_stats_enabled: bool, default = false /// Discover and retain file tasks during physical planning instead of execution. pub eager_file_planning: bool, default = false diff --git a/datafusion/distributed-iceberg/src/distributed_desired_task_count_handler.rs b/datafusion/distributed-iceberg/src/distributed_desired_task_count_handler.rs index e4a15ebe4..61f94c3ec 100644 --- a/datafusion/distributed-iceberg/src/distributed_desired_task_count_handler.rs +++ b/datafusion/distributed-iceberg/src/distributed_desired_task_count_handler.rs @@ -1,3 +1,4 @@ +use crate::common::scan_config::stats_from_snapshot; use std::num::NonZeroUsize; use datafusion::common::{Result, config_datafusion_err}; @@ -17,11 +18,21 @@ pub fn iceberg_desired_task_count( if !node.is::() && !node.is::() { return None; } - let statistics = match node.partition_statistics(None) { - Ok(statistics) => statistics, - Err(error) => return Some(Err(error)), + // Relation statistics may describe projected columns; parallelism uses whole-file work. + let total_bytes = if let Some(source) = node.downcast_ref::() { + source.planned_files().try_fold(0_usize, |bytes, task| { + bytes.checked_add(usize::try_from(task.length()).ok()?) + })? + } else { + let source = node.downcast_ref::()?; + source.feed.inner()?; + let statistics = + match stats_from_snapshot(source.table_snapshot.clone(), &source.scan.schema) { + Ok(statistics) => statistics, + Err(error) => return Some(Err(error)), + }; + *statistics.total_byte_size.get_value()? }; - let total_bytes = *statistics.total_byte_size.get_value()?; let config = DistributedConfig::from_session_config(ev.session_config).ok()?; Some( diff --git a/datafusion/distributed-iceberg/src/lazy/data_source.rs b/datafusion/distributed-iceberg/src/lazy/data_source.rs index 97339bab3..1f52ded6c 100644 --- a/datafusion/distributed-iceberg/src/lazy/data_source.rs +++ b/datafusion/distributed-iceberg/src/lazy/data_source.rs @@ -1,3 +1,4 @@ +use crate::common::scan_config::apply_column_statistics; use crate::common::scan_config::{ScanConfig, stats_from_snapshot}; use std::sync::Arc; @@ -180,7 +181,7 @@ impl DataSource for LazyIcebergDataSource { let mut stats = stats_from_snapshot(self.table_snapshot.clone(), &self.scan.schema)?; if let Some(col_stats) = &self.column_stats { - stats.column_statistics = col_stats.clone(); + apply_column_statistics(&mut stats, col_stats.clone()); } Ok(Arc::new(stats)) diff --git a/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs b/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs index 1328e63a7..2e51afe54 100644 --- a/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs +++ b/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs @@ -1,17 +1,143 @@ use crate::common::df_err; use datafusion::common::runtime::JoinSet; use datafusion::common::stats::Precision; -use datafusion::common::{ColumnStatistics, HashMap, Result}; +use datafusion::common::{ColumnStatistics, HashMap, HashSet, Result, exec_err}; use datafusion::error::DataFusionError; use datafusion::scalar::ScalarValue; use iceberg::puffin::APACHE_DATASKETCHES_THETA_V1; use iceberg::spec::{ - DataFile, Datum, ManifestContentType, ManifestList, PrimitiveLiteral, PrimitiveType, - SnapshotRef, + DataFile, Datum, ManifestContentType, ManifestList, NestedFieldRef, PrimitiveLiteral, + PrimitiveType, SnapshotRef, Type, }; use iceberg::table::Table; use std::sync::Arc; +use iceberg::scan::FileScanTask; + +/// Selected-file metrics, with snapshot NDV retained only as an inexact fallback. +/// FileScanTask does not retain manifest metrics, so this requires a metadata-only +/// second pass. Stop as soon as all selected files have been found. +pub(crate) async fn selected_column_statistics( + table: &Table, + snapshot: &SnapshotRef, + fields: &[NestedFieldRef], + files: &[FileScanTask], + may_filter_rows: bool, +) -> Result> { + if files.is_empty() || fields.is_empty() { + return Ok(vec![ + ColumnStatistics { + null_count: Precision::Exact(0), + distinct_count: Precision::Exact(0), + byte_size: Precision::Inexact(0), + ..ColumnStatistics::new_unknown() + }; + fields.len() + ]); + } + if files + .iter() + .any(|task| task.start() != 0 || task.length() != task.file_size_in_bytes()) + { + // Full-file metrics cannot describe a byte-range task. + return Ok(vec![ColumnStatistics::new_unknown(); fields.len()]); + } + let mut remaining: HashSet<_> = files.iter().map(|task| task.data_file_path()).collect(); + if remaining.len() != files.len() { + return exec_err!("Duplicate whole-file tasks in Iceberg column statistics lookup"); + } + let field_ids: Vec<_> = fields + .iter() + .map(|field| { + let mut ids = vec![]; + leaf_ids(field, &mut ids); + ids + }) + .collect(); + let mut statistics = vec![None; fields.len()]; + let manifests = table + .manifest_list_reader(snapshot) + .load() + .await + .map_err(df_err)?; + for manifest in manifests + .entries() + .iter() + .filter(|manifest| manifest.content == ManifestContentType::Data) + { + let manifest = table + .manifest_reader() + .read(manifest) + .await + .map_err(df_err)?; + for entry in manifest.entries().iter().filter(|entry| entry.is_alive()) { + let file = entry.data_file(); + if !remaining.remove(file.file_path()) { + continue; + } + for ((acc, field), ids) in statistics.iter_mut().zip(fields).zip(&field_ids) { + // Leaf metrics cannot describe a parent struct/list/map's value or null count. + let mut next = if matches!(field.field_type.as_ref(), Type::Primitive(_)) { + data_file_col_stats(file, field.id) + } else { + ColumnStatistics::new_unknown() + }; + next.byte_size = ids + .iter() + .try_fold(0_usize, |total, id| { + let size = usize::try_from(*file.column_sizes().get(id)?).ok()?; + total.checked_add(size) + }) + .map(Precision::Inexact) + .unwrap_or(Precision::Absent); + *acc = Some(merge_col_stats(acc.take(), next)); + } + } + if remaining.is_empty() { + break; + } + } + if !remaining.is_empty() { + return exec_err!("Planned Iceberg files missing from snapshot manifests"); + } + let rows = files.iter().try_fold(0_usize, |rows, task| { + rows.checked_add(usize::try_from(task.record_count()?).ok()?) + }); + let ndvs = ndv_from_metadata(table, snapshot.snapshot_id()); + Ok(statistics + .into_iter() + .zip(fields) + .map(|(stats, field)| { + let mut stats = stats.unwrap_or_else(ColumnStatistics::new_unknown); + if may_filter_rows { + stats.null_count = stats.null_count.to_inexact(); + } + // Snapshot sketches cannot be restricted to selected files. Retain their + // estimate, capped by the selected rows, without claiming exactness. + if let Some(&ndv) = ndvs.get(&field.id) { + stats.distinct_count = Precision::Inexact(rows.map_or(ndv, |rows| ndv.min(rows))); + } + stats + }) + .collect()) +} + +fn leaf_ids(field: &NestedFieldRef, ids: &mut Vec) { + match field.field_type.as_ref() { + Type::Primitive(_) | Type::Variant(_) => ids.push(field.id), + Type::Struct(fields) => { + for field in fields.fields() { + leaf_ids(field, ids); + } + } + Type::List(list) => leaf_ids(&list.element_field, ids), + Type::Map(map) => { + leaf_ids(&map.key_field, ids); + leaf_ids(&map.value_field, ids); + } + } +} + /// Getting number of distinct values for columns in a table /// /// Note: https://iceberg.apache.org/puffin-spec/#apache-datasketches-theta-v1-blob-type @@ -139,7 +265,13 @@ fn merge_col_stats(acc: Option, next: ColumnStatistics) -> Col null_count: acc.null_count.add(&next.null_count), min_value: acc.min_value.min(&next.min_value), max_value: acc.max_value.max(&next.max_value), - byte_size: acc.byte_size.add(&next.byte_size), + byte_size: acc + .byte_size + .get_value() + .zip(next.byte_size.get_value()) + .and_then(|(a, b)| a.checked_add(*b)) + .map(Precision::Inexact) + .unwrap_or(Precision::Absent), sum_value: Precision::Absent, distinct_count: Precision::Absent, }, diff --git a/datafusion/distributed-iceberg/src/table_provider/scan.rs b/datafusion/distributed-iceberg/src/table_provider/scan.rs index aeeb2ec94..8a70b4e93 100644 --- a/datafusion/distributed-iceberg/src/table_provider/scan.rs +++ b/datafusion/distributed-iceberg/src/table_provider/scan.rs @@ -1,8 +1,8 @@ use crate::common::df_err; -use crate::common::scan_config::ScanConfig; +use crate::common::scan_config::{ScanConfig, apply_column_statistics}; use crate::eager::assign_files; use crate::eager::planned_files::PlannedFiles; -use crate::table_provider::column_statistics::compute_column_stats; +use crate::table_provider::column_statistics::{compute_column_stats, selected_column_statistics}; use crate::{EagerIcebergDataSource, IcebergConfig, IcebergWorkUnitFeed, LazyIcebergDataSource}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{Result, config_err, exec_err}; @@ -92,27 +92,44 @@ pub(crate) async fn plan_scan( || planned .as_ref() .is_some_and(|files| files.tasks().iter().any(|task| !task.deletes().is_empty())); - let column_stats = if IcebergConfig::from_task_context(&context).column_stats_enabled - && !(planned.is_some() && may_filter_rows) - { + let column_stats = if IcebergConfig::from_task_context(&context).column_stats_enabled { let schema = match &table_snapshot { Some(snap) => snap.schema(table.metadata()).map_err(df_err)?, // empty table None => table.metadata().current_schema().clone(), }; - let fields = schema.as_struct().fields().to_vec(); - let field_ids: Vec = match opts.projection { - Some(projection) => projection.iter().map(|&idx| fields[idx].id).collect(), - None => fields.iter().map(|f| f.id).collect(), + let fields = schema.as_struct().fields(); + let fields: Vec<_> = match opts.projection { + Some(projection) => projection + .iter() + .map(|&idx| Arc::clone(&fields[idx])) + .collect(), + None => fields.to_vec(), }; - Some(compute_column_stats(table.clone(), field_ids, table_snapshot.clone()).await?) + if let Some(files) = &planned + && let Some(snapshot) = &table_snapshot + { + Some( + selected_column_statistics( + &table, + snapshot, + &fields, + files.tasks(), + may_filter_rows, + ) + .await?, + ) + } else { + let field_ids = fields.iter().map(|field| field.id).collect(); + Some(compute_column_stats(table.clone(), field_ids, table_snapshot.clone()).await?) + } } else { None }; if let Some(planned) = planned { let mut statistics = planned.statistics(&scan.schema, may_filter_rows); if let Some(column_stats) = column_stats { - statistics.column_statistics = column_stats; + apply_column_statistics(&mut statistics, column_stats); } let partitions = assign_files(planned.tasks().iter(), 1, partitioning.partition_count())? .pop() diff --git a/datafusion/distributed-iceberg/tests/desired_task_count.rs b/datafusion/distributed-iceberg/tests/desired_task_count.rs index 90f792a31..99e9fa6f8 100644 --- a/datafusion/distributed-iceberg/tests/desired_task_count.rs +++ b/datafusion/distributed-iceberg/tests/desired_task_count.rs @@ -199,6 +199,8 @@ mod tests { let mut builder = IcebergTestHarness::builder() .with_session_extension(|state| state.with_iceberg_integration(Default::default())) .with_table(create_taxi_table().await?) + // Column statistics must not change how much work the estimator sees. + .with_config_option("iceberg.column_stats_enabled", true) .configure_session(|mut state| { let config = state.config().get_or_insert_default(); config.options_mut().execution.target_partitions = 2; diff --git a/datafusion/distributed-iceberg/tests/nested_column_statistics.rs b/datafusion/distributed-iceberg/tests/nested_column_statistics.rs new file mode 100644 index 000000000..39c78d9b6 --- /dev/null +++ b/datafusion/distributed-iceberg/tests/nested_column_statistics.rs @@ -0,0 +1,132 @@ +#[cfg(test)] +mod tests { + use std::error::Error; + use std::sync::Arc; + + use datafusion::common::stats::Precision; + use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_plan::statistics::{StatisticsArgs, StatisticsContext}; + use datafusion_distributed_iceberg::IcebergExt; + use datafusion_distributed_iceberg_test_tools::{FixtureCatalog, IcebergTestHarness}; + use iceberg::TableIdent; + use iceberg::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, NestedField, + PrimitiveType, Schema, Struct, StructType, Type, + }; + + #[tokio::test] + async fn selected_nested_columns_sum_leaves_and_keep_missing_leaves_unknown() + -> Result<(), Box> { + let harness = nested_fixture().await?; + for (id, expected) in [(1, Precision::Inexact(300)), (2, Precision::Absent)] { + let plan = harness + .physical_plan(&format!("SELECT details FROM nested WHERE id = {id}")) + .await?; + let mut checked = false; + plan.apply(|node| { + if node.is::() { + let stats = + StatisticsContext::new().compute(node.as_ref(), &StatisticsArgs::new())?; + let column = node.schema().index_of("details")?; + assert_eq!(stats.column_statistics[column].byte_size, expected); + assert_eq!( + stats.column_statistics[column].null_count, + Precision::Absent + ); + assert_eq!(stats.column_statistics[column].min_value, Precision::Absent); + assert_eq!(stats.column_statistics[column].max_value, Precision::Absent); + assert_eq!(stats.num_rows, Precision::Inexact(10)); + checked = true; + } + Ok(TreeNodeRecursion::Continue) + })?; + assert!(checked); + } + Ok(()) + } + + // A table of two synthetic whole files that are never opened. The first sizes every + // leaf of `details`; the second has no size for the list element, so the struct's + // size cannot be reconstructed from its leaves. + async fn nested_fixture() -> Result> { + let schema = Schema::builder() + .with_fields([ + Arc::new(NestedField::required( + 1, + "id", + Type::Primitive(PrimitiveType::Int), + )), + Arc::new(NestedField::optional( + 2, + "details", + Type::Struct(StructType::new(vec![ + Arc::new(NestedField::optional( + 3, + "name", + Type::Primitive(PrimitiveType::String), + )), + Arc::new(NestedField::optional( + 4, + "values", + Type::List(ListType::new(Arc::new(NestedField::optional( + 5, + "element", + Type::Primitive(PrimitiveType::Long), + )))), + )), + ])), + )), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let table = fixtures + .table(TableIdent::from_strs(["default", "nested"])?, schema) + .create() + .await?; + // Field ids as the catalog assigned them when creating the table. + let schema = Arc::clone(table.table().metadata().current_schema()); + let field_id = |name: &str| { + schema + .field_id_by_name(name) + .ok_or_else(|| format!("no field {name}")) + }; + let (id, name, element) = ( + field_id("id")?, + field_id("details.name")?, + field_id("details.values.element")?, + ); + let location = table.table().metadata().location().to_owned(); + let mut files = Vec::new(); + for file in [1, 2] { + let sizes = if file == 1 { + [(id, 40), (name, 100), (element, 200)] + .into_iter() + .collect() + } else { + [(id, 40), (name, 900)].into_iter().collect() + }; + files.push( + DataFileBuilder::default() + .content(DataContentType::Data) + .file_format(DataFileFormat::Parquet) + .file_path(format!("{location}/data/unopened-nested-{file}.parquet")) + .partition(Struct::empty()) + .record_count(10) + .file_size_in_bytes(1000) + .column_sizes(sizes) + .lower_bounds([(id, Datum::int(file))].into()) + .upper_bounds([(id, Datum::int(file))].into()) + .build()?, + ); + } + Ok(IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(table) + .with_data_files(files) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.column_stats_enabled", true) + .build() + .await?) + } +} diff --git a/datafusion/distributed-iceberg/tests/statistics.rs b/datafusion/distributed-iceberg/tests/statistics.rs index e7b711412..6b7241fd5 100644 --- a/datafusion/distributed-iceberg/tests/statistics.rs +++ b/datafusion/distributed-iceberg/tests/statistics.rs @@ -13,18 +13,27 @@ mod tests { use datafusion::arrow::array::record_batch; // `record_batch!` refers to `arrow_schema` by name. use datafusion::arrow::datatypes as arrow_schema; + use datafusion::common::internal_datafusion_err; use datafusion::common::stats::Precision; + use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion::common::{ColumnStatistics, Statistics}; + use datafusion::datasource::source::DataSourceExec; use datafusion::error::Result; use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties}; use datafusion::parquet::schema::types::ColumnPath; + use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::displayable; use datafusion::physical_plan::statistics::{StatisticsArgs, StatisticsContext}; use datafusion::scalar::ScalarValue; + use datafusion_distributed::{DesiredTaskCountEvent, DistributedExt}; + use datafusion_distributed_iceberg::iceberg_desired_task_count; + use datafusion_distributed_iceberg_test_tools::IcebergTestHarnessBuilder; use iceberg::TableIdent; + use iceberg::puffin::APACHE_DATASKETCHES_THETA_V1; use iceberg::spec::{ - DataFile, MAIN_BRANCH, NestedField, Operation, PrimitiveType, Schema, Snapshot, Summary, - TableMetadata, Type, + BlobMetadata, DataContentType, DataFile, DataFileBuilder, DataFileFormat, Datum, Literal, + MAIN_BRANCH, NestedField, Operation, PrimitiveType, Schema, Snapshot, StatisticsFile, + Struct, Summary, TableMetadata, Type, }; // Values from the generated taxi snapshot summary. @@ -142,13 +151,14 @@ mod tests { let stats = query_statistics(&harness, "SELECT * FROM metrics").await?; assert_eq!(stats.num_rows, Precision::Exact(METRICS_ROWS)); + // With a size for every column, summed column bytes replace whole-file bytes. assert_eq!( stats.total_byte_size, - Precision::Exact( + Precision::Inexact( files .iter() - .map(|file| file.file_size_in_bytes() as usize) - .sum() + .flat_map(|file| file.column_sizes().values()) + .sum::() as usize ) ); assert_eq!( @@ -456,6 +466,269 @@ mod tests { Ok((table, files)) } + // ---- Selected-file (eager) column statistics over synthetic manifest metrics ---- + // + // These fixtures describe files through manifest entries alone; the data paths are + // never opened. Metric keys are field ids: 1 is vendor_id and 4 is passenger_count, + // not schema indexes. trip_distance (id 5) has no metrics. The snapshot keeps the + // generated taxi summary, so summary totals disagree with the synthetic files and + // selected-file metrics are distinguishable. + + // Each synthetic file's record count and size; two files replace the taxi manifest. + const SYNTHETIC_ROWS: usize = 87_500; + const SYNTHETIC_BYTES: usize = 2_240_191; + + #[tokio::test] + async fn eager_full_scan_merges_selected_file_metrics() -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let stats = query_statistics(&harness, "SELECT * FROM taxi").await?; + assert_eq!(stats.num_rows, Precision::Exact(2 * SYNTHETIC_ROWS)); + assert_eq!(stats.total_byte_size, Precision::Exact(2 * SYNTHETIC_BYTES)); + let mut columns = vec![ColumnStatistics::new_unknown(); 13]; + columns[0] = expected_vendor_id_statistics(); + columns[3] = expected_passenger_count_statistics(); + assert_eq!(stats.column_statistics, columns); + Ok(()) + } + + #[tokio::test] + async fn eager_projection_reorders_selected_file_metrics() -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let stats = query_statistics( + &harness, + "SELECT passenger_count, vendor_id, trip_distance FROM taxi", + ) + .await?; + assert_eq!(stats.num_rows, Precision::Exact(2 * SYNTHETIC_ROWS)); + assert_eq!( + stats.column_statistics, + vec![ + expected_passenger_count_statistics(), + expected_vendor_id_statistics(), + ColumnStatistics::new_unknown(), + ] + ); + Ok(()) + } + + #[tokio::test] + async fn filtered_statistics_use_selected_files_and_cap_snapshot_ndv() + -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + // Manifest bounds select only the first file: passenger_count spans 10 to 20. + let source = scan_node( + &harness + .physical_plan("SELECT vendor_id FROM taxi WHERE passenger_count < 25") + .await?, + )?; + let stats = StatisticsContext::new().compute(source.as_ref(), &StatisticsArgs::new())?; + let column = source.schema().index_of("vendor_id")?; + assert_eq!(stats.num_rows, Precision::Inexact(SYNTHETIC_ROWS)); + // First file: 100 bytes. A retained-row fraction would incorrectly yield 200. + assert_eq!( + stats.column_statistics[column].byte_size, + Precision::Inexact(100) + ); + assert_eq!( + stats.column_statistics[column].min_value, + Precision::Inexact(ScalarValue::Int32(Some(2))) + ); + assert_eq!( + stats.column_statistics[column].max_value, + Precision::Inexact(ScalarValue::Int32(Some(9))) + ); + // The snapshot sketch claims 100,000 distinct values; the selected rows cap it. + assert_eq!( + stats.column_statistics[column].distinct_count, + Precision::Inexact(SYNTHETIC_ROWS) + ); + assert_eq!( + stats.column_statistics[column].null_count, + Precision::Inexact(2) + ); + // Both scan columns have sizes: vendor_id 100 + passenger_count 200. + assert_eq!(stats.total_byte_size, Precision::Inexact(300)); + Ok(()) + } + + #[tokio::test] + async fn incomplete_filtered_metrics_keep_unknown_columns_and_file_byte_fallback() + -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let source = scan_node( + &harness + .physical_plan("SELECT trip_distance FROM taxi WHERE passenger_count < 25") + .await?, + )?; + let stats = StatisticsContext::new().compute(source.as_ref(), &StatisticsArgs::new())?; + let column = source.schema().index_of("trip_distance")?; + assert_eq!(stats.column_statistics[column].byte_size, Precision::Absent); + // An unknown column size keeps the selected file's total bytes instead. + assert_eq!(stats.total_byte_size, Precision::Inexact(SYNTHETIC_BYTES)); + Ok(()) + } + + #[tokio::test] + async fn filtered_scan_merges_bounds_and_preserves_missing_null_counts() + -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let source = scan_node( + &harness + .physical_plan( + "SELECT passenger_count, vendor_id FROM taxi WHERE passenger_count < 100", + ) + .await?, + )?; + let stats = StatisticsContext::new().compute(source.as_ref(), &StatisticsArgs::new())?; + let passenger = &stats.column_statistics[source.schema().index_of("passenger_count")?]; + assert_eq!( + passenger.min_value, + Precision::Inexact(ScalarValue::Int64(Some(10))) + ); + assert_eq!( + passenger.max_value, + Precision::Inexact(ScalarValue::Int64(Some(40))) + ); + // One file omits this column's null count: the total must stay unknown. + assert_eq!(passenger.null_count, Precision::Absent); + assert_eq!(passenger.distinct_count, Precision::Absent); + let vendor = &stats.column_statistics[source.schema().index_of("vendor_id")?]; + assert_eq!(vendor.null_count, Precision::Inexact(5)); + assert_eq!(vendor.distinct_count, Precision::Inexact(100_000)); + Ok(()) + } + + #[tokio::test] + async fn projected_bytes_do_not_reduce_scan_task_count() -> Result<(), Box> { + let harness = harness_with_manifest_metrics().await?; + let source = scan_node( + &harness + .physical_plan("SELECT vendor_id FROM taxi WHERE passenger_count < 100") + .await?, + )?; + let stats = StatisticsContext::new().compute(source.as_ref(), &StatisticsArgs::new())?; + assert_eq!(stats.total_byte_size, Precision::Inexact(1000)); + // 4.48 MB of file work / 1 MB / 1 partition, capped at the two files. + let estimate = iceberg_desired_task_count(DesiredTaskCountEvent { + plan: &source, + session_config: harness.state().config(), + }) + .transpose()? + .map(|response| response.task_count.as_usize()); + assert_eq!(estimate, Some(2)); + Ok(()) + } + + fn scan_node(plan: &Arc) -> Result> { + let mut source = None; + plan.apply(|node| { + if node.is::() { + source = Some(Arc::clone(node)); + return Ok(TreeNodeRecursion::Stop); + } + Ok(TreeNodeRecursion::Continue) + })?; + source.ok_or_else(|| internal_datafusion_err!("expected an Iceberg scan")) + } + + // Planning-only eager fixture over the taxi table's current snapshot. + async fn harness_with_manifest_metrics() -> Result> { + Ok(manifest_metrics_builder() + .await? + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.column_stats_enabled", true) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + state.with_distributed_file_scan_config_bytes_per_partition(1_000_000) + })? + .build() + .await?) + } + + async fn manifest_metrics_builder() -> Result> { + let table = create_taxi_table().await?; + let location = table.table().metadata().location().to_owned(); + let mut file = DataFileBuilder::default(); + file.content(DataContentType::Data) + .file_format(DataFileFormat::Parquet) + .record_count(SYNTHETIC_ROWS as u64) + .file_size_in_bytes(SYNTHETIC_BYTES as u64) + .partition(Struct::from_iter([Some(Literal::date_from_str( + "2024-01-10", + )?)])); + let first = file + .file_path(format!("{location}/data/unopened-metrics-first.parquet")) + .null_value_counts([(1, 2), (4, 4)].into()) + .column_sizes([(1, 100), (4, 200)].into()) + .lower_bounds([(1, Datum::int(2)), (4, Datum::long(10))].into()) + .upper_bounds([(1, Datum::int(9)), (4, Datum::long(20))].into()) + .build()?; + let second = file + .file_path(format!("{location}/data/unopened-metrics-second.parquet")) + .null_value_counts([(1, 3)].into()) + .column_sizes([(1, 300), (4, 400)].into()) + .lower_bounds([(1, Datum::int(1)), (4, Datum::long(30))].into()) + .upper_bounds([(1, Datum::int(5)), (4, Datum::long(40))].into()) + .build()?; + Ok(IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(table) + .with_metadata(vendor_id_ndv_sketch) + .with_data_files([first, second])) + } + + // Attaches a snapshot statistics file claiming 100,000 distinct vendor ids. + fn vendor_id_ndv_sketch(metadata: &TableMetadata) -> TableMetadata { + let snapshot = metadata.current_snapshot().expect("taxi has a snapshot"); + let statistics = StatisticsFile { + snapshot_id: snapshot.snapshot_id(), + statistics_path: format!( + "{}/metadata/unopened-statistics.puffin", + metadata.location() + ), + file_size_in_bytes: 0, + file_footer_size_in_bytes: 0, + key_metadata: None, + blob_metadata: vec![BlobMetadata { + r#type: APACHE_DATASKETCHES_THETA_V1.to_owned(), + snapshot_id: snapshot.snapshot_id(), + sequence_number: snapshot.sequence_number(), + fields: vec![1], + properties: [("ndv".to_owned(), "100000".to_owned())].into(), + }], + }; + metadata + .clone() + .into_builder(None) + .set_statistics(statistics) + .build() + .expect("taxi metadata stays valid") + .metadata + } + + // The merged vendor_id metrics of both synthetic files, including the capped sketch. + fn expected_vendor_id_statistics() -> ColumnStatistics { + ColumnStatistics { + null_count: Precision::Exact(5), + distinct_count: Precision::Inexact(100_000), + min_value: Precision::Inexact(ScalarValue::Int32(Some(1))), + max_value: Precision::Inexact(ScalarValue::Int32(Some(9))), + byte_size: Precision::Inexact(400), + ..ColumnStatistics::new_unknown() + } + } + + fn expected_passenger_count_statistics() -> ColumnStatistics { + ColumnStatistics { + min_value: Precision::Inexact(ScalarValue::Int64(Some(10))), + max_value: Precision::Inexact(ScalarValue::Int64(Some(40))), + byte_size: Precision::Inexact(600), + // One file omits this column's null count: the total must stay unknown. + ..ColumnStatistics::new_unknown() + } + } + // The column statistics of the metrics table, in schema order, with column stats enabled. fn expected_column_statistics(files: &[DataFile]) -> [ColumnStatistics; 3] { let size = |field_id| { From 1d666e24d74b97d82f31a867d57374392717bdd5 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Thu, 29 Oct 2026 16:49:11 -0400 Subject: [PATCH 2/4] Export ColumnByteStatisticsProvider for dd-source Exports datafusion_distributed::ColumnByteStatisticsProvider so an embedder can register it in a StatisticsRegistry and keep column-byte estimates through grouping keys and inner joins. It is not registered automatically by Iceberg. (cherry picked from commit 29248e717813cd1d5a70f471a43842b97cc116be) --- datafusion/distributed-iceberg/README.md | 8 + .../src/distributed_planner/mod.rs | 1 + .../statistics/column_byte_statistics.rs | 321 ++++++++++++++++++ .../src/distributed_planner/statistics/mod.rs | 3 + datafusion/distributed/src/lib.rs | 4 +- 5 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 datafusion/distributed/src/distributed_planner/statistics/column_byte_statistics.rs diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index ce512f9dd..349a0fc33 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -112,6 +112,14 @@ files and can overestimate the filtered result. Sum statistics remain unknown. Another manifest pass is needed because the pinned Iceberg API does not retain column metrics in file tasks. Scan task counts continue to use whole-file bytes. +The Datadog integration also exports `datafusion_distributed::ColumnByteStatisticsProvider`. +To preserve available column-byte estimates through grouping keys and inner joins, +register it with `SessionStateBuilder::with_statistics_registry` using +`StatisticsRegistry::with_providers(vec![Arc::new(ColumnByteStatisticsProvider)])`, +and enable `datafusion.optimizer.use_statistics_registry`. The provider retains the +operators' normal row-count estimates; compressed column bytes remain a heuristic, +not a measurement of join memory. It is not registered automatically by Iceberg. + ```bash cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools ``` diff --git a/datafusion/distributed/src/distributed_planner/mod.rs b/datafusion/distributed/src/distributed_planner/mod.rs index ff32d02f7..eca178a33 100644 --- a/datafusion/distributed/src/distributed_planner/mod.rs +++ b/datafusion/distributed/src/distributed_planner/mod.rs @@ -21,4 +21,5 @@ pub(crate) use inject_network_boundaries::{ }; pub use network_boundary::{NetworkBoundary, NetworkBoundaryExt, ProducerHead}; pub use session_state_builder_ext::SessionStateBuilderExt; +pub use statistics::ColumnByteStatisticsProvider; pub(crate) use statistics::calculate_cost; diff --git a/datafusion/distributed/src/distributed_planner/statistics/column_byte_statistics.rs b/datafusion/distributed/src/distributed_planner/statistics/column_byte_statistics.rs new file mode 100644 index 000000000..5981f1dd9 --- /dev/null +++ b/datafusion/distributed/src/distributed_planner/statistics/column_byte_statistics.rs @@ -0,0 +1,321 @@ +use std::sync::Arc; + +use datafusion::common::stats::Precision; +use datafusion::common::{JoinSide, JoinType, Result, internal_err}; +use datafusion::physical_expr::expressions::Column; +use datafusion::physical_plan::aggregates::AggregateExec; +use datafusion::physical_plan::joins::{HashJoinExec, utils::build_join_schema}; +use datafusion::physical_plan::operator_statistics::{ + ExtendedStatistics, StatisticsProvider, StatisticsResult, +}; +use datafusion::physical_plan::{ExecutionPlan, StatisticsArgs}; + +/// Propagates available column byte estimates without changing the operators' +/// row-count, distinct-count or value-statistics models. +/// +/// Direct grouping keys and inner hash-join columns retain their average input +/// width, scaled to the estimated output row count. Missing sizes stay unknown. +/// Their totals can be recovered when every output column has a size. Grouping sets, +/// computed group keys and aggregate result columns are left to DataFusion. +/// +/// Register through [`StatisticsRegistry::with_providers`] and enable +/// `datafusion.optimizer.use_statistics_registry`. A registry containing only +/// this provider uses the operators' normal cardinality estimates rather than +/// the registry's alternative built-in join estimator. +/// +/// This is a uniform-width heuristic, not a memory measurement: in particular, +/// compressed Iceberg column bytes remain compressed-byte estimates. +/// +/// [`StatisticsRegistry::with_providers`]: datafusion::physical_plan::operator_statistics::StatisticsRegistry::with_providers +#[derive(Debug, Default)] +pub struct ColumnByteStatisticsProvider; + +impl StatisticsProvider for ColumnByteStatisticsProvider { + fn compute_statistics( + &self, + plan: &dyn ExecutionPlan, + children: &[ExtendedStatistics], + ) -> Result { + let inputs: Vec<_> = children + .iter() + .map(|stats| Arc::clone(stats.base_arc())) + .collect(); + let mut stats = + Arc::unwrap_or_clone(plan.statistics_from_inputs(&inputs, &StatisticsArgs::new())?); + let mut repaired_columns = false; + if let Some(aggregate) = plan.downcast_ref::() { + let groups = aggregate.group_expr(); + if groups.groups().len() == 1 && groups.groups()[0].iter().all(|is_null| !is_null) { + for (output, (expr, _)) in groups.expr().iter().enumerate() { + if let Some(column) = expr.downcast_ref::() { + stats.column_statistics[output].byte_size = scale_bytes( + inputs[0].column_statistics[column.index()].byte_size, + inputs[0].num_rows, + stats.num_rows, + ); + repaired_columns = true; + } + } + } + } + if let Some(join) = plan.downcast_ref::() + && *join.join_type() == JoinType::Inner + { + let (_, columns) = build_join_schema( + join.left().schema().as_ref(), + join.right().schema().as_ref(), + join.join_type(), + ); + for (output, column_stats) in stats.column_statistics.iter_mut().enumerate() { + let index = join + .projection + .as_ref() + .map_or(output, |projection| projection[output]); + let column = &columns[index]; + let input = match column.side { + JoinSide::Left => &inputs[0], + JoinSide::Right => &inputs[1], + JoinSide::None => { + return internal_err!("Unexpected synthetic inner-join column"); + } + }; + column_stats.byte_size = scale_bytes( + input.column_statistics[column.index].byte_size, + input.num_rows, + stats.num_rows, + ); + } + repaired_columns = true; + } + if repaired_columns + && let Some(bytes) = stats + .column_statistics + .iter() + .try_fold(0_usize, |total, column| { + total.checked_add(*column.byte_size.get_value()?) + }) + { + stats.total_byte_size = Precision::Inexact(bytes); + } + Ok(StatisticsResult::Computed(ExtendedStatistics::new(stats))) + } +} + +fn scale_bytes( + bytes: Precision, + input_rows: Precision, + output_rows: Precision, +) -> Precision { + let Some(output) = output_rows.get_value() else { + return Precision::Absent; + }; + if *output == 0 { + return Precision::Inexact(0); + } + let (Some(bytes), Some(input)) = (bytes.get_value(), input_rows.get_value()) else { + return Precision::Absent; + }; + if *input == 0 { + return Precision::Absent; + } + let scaled = ((*bytes as u128) * (*output as u128)).div_ceil(*input as u128); + usize::try_from(scaled) + .map(Precision::Inexact) + .unwrap_or(Precision::Absent) +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::common::{NullEquality, Statistics}; + use datafusion::execution::SessionStateBuilder; + use datafusion::physical_expr::PhysicalExpr; + use datafusion::physical_optimizer::{PhysicalOptimizerRule, join_selection::JoinSelection}; + use datafusion::physical_plan::aggregates::{AggregateMode, PhysicalGroupBy}; + use datafusion::physical_plan::joins::PartitionMode; + use datafusion::physical_plan::operator_statistics::StatisticsRegistry; + use datafusion::physical_plan::statistics::StatisticsContext; + use datafusion::physical_plan::test::exec::StatisticsExec; + use datafusion::prelude::SessionConfig; + + #[test] + fn group_keys_keep_bytes_without_retaining_payload_cost() -> Result<()> { + let aggregate = distinct_keys(source(10_000, 32, true))?; + let ordinary = + StatisticsContext::new().compute(aggregate.as_ref(), &StatisticsArgs::new())?; + let stats = registry().compute_base(aggregate.as_ref())?; + assert_eq!(stats.num_rows, ordinary.num_rows); + assert_eq!( + stats.column_statistics[0].distinct_count, + ordinary.column_statistics[0].distinct_count + ); + assert_eq!(ordinary.column_statistics[0].byte_size, Precision::Absent); + assert_eq!( + stats.column_statistics[0].byte_size, + Precision::Inexact(160_000) + ); + assert_eq!(stats.total_byte_size, Precision::Inexact(160_000)); + Ok(()) + } + + #[test] + fn join_selection_can_build_derived_keys_instead_of_fewer_wide_rows() -> Result<()> { + let keys = inner_join( + distinct_keys(source(10_000, 32, true))?, + source(5_000, 32, true), + )?; + let join = Arc::new(HashJoinExec::try_new( + source(1_000, 32, true), + keys, + vec![(key(), key())], + None, + &JoinType::LeftSemi, + None, + PartitionMode::Partitioned, + NullEquality::NullEqualsNothing, + false, + )?) as Arc; + for (enabled, expected) in [(false, JoinType::LeftSemi), (true, JoinType::RightSemi)] { + let mut config = SessionConfig::new(); + config.options_mut().optimizer.use_statistics_registry = enabled; + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_statistics_registry(registry()) + .build(); + let optimized = + JoinSelection::new().optimize_with_context(Arc::clone(&join), &state)?; + assert_eq!( + optimized + .downcast_ref::() + .unwrap() + .join_type(), + &expected + ); + assert_eq!(optimized.schema(), join.schema()); + } + Ok(()) + } + + #[test] + fn missing_sizes_are_not_invented_and_known_source_totals_are_preserved() -> Result<()> { + let input = source(10_000, 32, false); + let original = StatisticsContext::new().compute(input.as_ref(), &StatisticsArgs::new())?; + assert_eq!( + registry().compute_base(input.as_ref())?.total_byte_size, + original.total_byte_size + ); + let keys = inner_join(distinct_keys(input)?, source(5_000, 32, true))?; + let stats = registry().compute_base(keys.as_ref())?; + assert_eq!(stats.column_statistics[0].byte_size, Precision::Absent); + assert_eq!(stats.total_byte_size, Precision::Absent); + Ok(()) + } + + #[test] + fn outer_joins_do_not_get_totals_from_unscaled_input_columns() -> Result<()> { + for join_type in [JoinType::Left, JoinType::Right, JoinType::Full] { + let join = HashJoinExec::try_new( + source(100, 32, true), + source(200, 32, true), + vec![(key(), key())], + None, + &join_type, + None, + PartitionMode::Partitioned, + NullEquality::NullEqualsNothing, + false, + )?; + assert_eq!( + registry().compute_base(&join)?.total_byte_size, + Precision::Absent + ); + } + Ok(()) + } + + #[test] + fn scaling_handles_expansion_empty_inputs_unknowns_and_overflow() { + assert_eq!( + scale_bytes( + Precision::Exact(10), + Precision::Exact(3), + Precision::Inexact(10) + ), + Precision::Inexact(34) + ); + assert_eq!( + scale_bytes(Precision::Absent, Precision::Exact(3), Precision::Exact(0)), + Precision::Inexact(0) + ); + assert_eq!( + scale_bytes( + Precision::Exact(10), + Precision::Exact(0), + Precision::Inexact(10) + ), + Precision::Absent + ); + assert_eq!( + scale_bytes( + Precision::Exact(usize::MAX), + Precision::Exact(1), + Precision::Inexact(2) + ), + Precision::Absent + ); + } + + fn registry() -> StatisticsRegistry { + StatisticsRegistry::with_providers(vec![Arc::new(ColumnByteStatisticsProvider)]) + } + + fn source(rows: usize, key_width: usize, known: bool) -> Arc { + let schema = Schema::new(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("payload", DataType::Utf8, true), + ]); + let mut stats = Statistics::new_unknown(&schema); + stats.num_rows = Precision::Inexact(rows); + stats.total_byte_size = Precision::Inexact(rows * (key_width + 1024)); + stats.column_statistics[0].distinct_count = Precision::Inexact(rows.min(5_000)); + if known { + stats.column_statistics[0].byte_size = Precision::Inexact(rows * key_width); + stats.column_statistics[1].byte_size = Precision::Inexact(rows * 1024); + } + Arc::new(StatisticsExec::new(stats, schema)) + } + + fn key() -> Arc { + Arc::new(Column::new("key", 0)) + } + + fn distinct_keys(input: Arc) -> Result> { + Ok(Arc::new(AggregateExec::try_new( + AggregateMode::Single, + PhysicalGroupBy::new_single(vec![(key(), "key".to_owned())]), + vec![], + vec![], + Arc::clone(&input), + input.schema(), + )?)) + } + + fn inner_join( + left: Arc, + right: Arc, + ) -> Result> { + Ok(Arc::new(HashJoinExec::try_new( + left, + right, + vec![(key(), key())], + None, + &JoinType::Inner, + Some(vec![0]), + PartitionMode::Partitioned, + NullEquality::NullEqualsNothing, + false, + )?)) + } +} diff --git a/datafusion/distributed/src/distributed_planner/statistics/mod.rs b/datafusion/distributed/src/distributed_planner/statistics/mod.rs index 59f1ec04c..68ae31d87 100644 --- a/datafusion/distributed/src/distributed_planner/statistics/mod.rs +++ b/datafusion/distributed/src/distributed_planner/statistics/mod.rs @@ -1,3 +1,4 @@ +mod column_byte_statistics; mod complexity; mod complexity_cpu; mod complexity_memory; @@ -6,5 +7,7 @@ mod cost; mod default_bytes_for_datatype; mod plan_statistics; +pub use column_byte_statistics::ColumnByteStatisticsProvider; + #[allow(unused)] // will be used in a follow-up PR. pub(crate) use cost::calculate_cost; diff --git a/datafusion/distributed/src/lib.rs b/datafusion/distributed/src/lib.rs index d4d7a401a..2680e6fe5 100644 --- a/datafusion/distributed/src/lib.rs +++ b/datafusion/distributed/src/lib.rs @@ -22,8 +22,8 @@ pub use arrow_ipc::CompressionType; pub use coordinator::DistributedExec; pub use distributed_ext::{DistributedExt, DistributedGetterExt}; pub use distributed_planner::{ - DistributedConfig, NetworkBoundary, NetworkBoundaryExt, ProducerHead, SessionStateBuilderExt, - create_distributed_plan, + ColumnByteStatisticsProvider, DistributedConfig, NetworkBoundary, NetworkBoundaryExt, + ProducerHead, SessionStateBuilderExt, create_distributed_plan, }; pub use dynamic_filtering::rewrite_distributed_plan_with_dynamic_filters; pub use events::{ From 78f47d14c3042bc0c661a601e7091eafc42bceb0 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Thu, 5 Nov 2026 16:24:00 +0000 Subject: [PATCH 3/4] Reuse planned file metrics for eager Iceberg column statistics Request column statistics from the scan (iceberg-rust with_column_stats, apache/iceberg-rust#3424) and summarize them from the planned FileScanTasks instead of re-reading manifests, so a scan reads its manifests once. Point iceberg and iceberg-storage-opendal at DataDog/iceberg-rust 9354b42 (upstream main on Arrow 59), the revision dd-source uses, and patch datafusion-iceberg's apache/iceberg-rust dependency to the same source so FileScanTask is one type. The retained metrics stay on each task, so they are also serialized to workers with the file list and counted by the planning memory reservation: iceberg-rust has no way to drop them from a planned task. (cherry picked from commit db8db2560fe7eea29b16296b6ca56b8afab8ee3c) --- Cargo.lock | 125 ++++++++++++- Cargo.toml | 5 + .../distributed-iceberg-test-tools/Cargo.toml | 4 +- datafusion/distributed-iceberg/Cargo.toml | 8 +- datafusion/distributed-iceberg/README.md | 10 +- datafusion/distributed-iceberg/src/config.rs | 2 +- .../src/eager/planned_files.rs | 32 +++- .../src/table_provider/column_statistics.rs | 166 ++++++++++-------- .../src/table_provider/scan.rs | 62 ++++--- 9 files changed, 304 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d72ad4fb..cac5ef273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3612,6 +3612,27 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777d18aa0f12f8b285331cd867133ee14422b3f023f6d388034c47d43e28786a" +dependencies = [ + "approx", + "num-traits", + "serde", + "thiserror 2.0.20", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -4043,7 +4064,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" dependencies = [ "aes-gcm", "anyhow", @@ -4051,6 +4072,7 @@ dependencies = [ "array-init", "arrow-arith", "arrow-array", + "arrow-avro", "arrow-buffer", "arrow-cast", "arrow-ord", @@ -4078,6 +4100,7 @@ dependencies = [ "once_cell", "ordered-float", "parquet", + "parquet-geospatial", "rand 0.9.5", "reqwest 0.12.28", "roaring", @@ -4119,7 +4142,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" dependencies = [ "proc-macro2", "quote", @@ -4129,7 +4152,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" dependencies = [ "anyhow", "async-trait", @@ -5118,6 +5141,28 @@ dependencies = [ "libm", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num_threads" version = "0.1.7" @@ -5567,6 +5612,7 @@ dependencies = [ "num-integer", "num-traits", "object_store", + "parquet-geospatial", "ring", "seq-macro", "simdutf8", @@ -5576,6 +5622,19 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "parquet-geospatial" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634827365851180ca26bf54c76e970dc1c14c075684c0580ee97d8d0c953faf1" +dependencies = [ + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "wkb", +] + [[package]] name = "pbkdf2" version = "0.12.2" @@ -5912,6 +5971,15 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -7821,6 +7889,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.19+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd43688c4858d7264ae1cf9a9f874a67c3feb74a883b1cd7bebd6a4ea8307218" +dependencies = [ + "indexmap 2.14.2", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "tonic" version = "0.14.6" @@ -8642,12 +8740,33 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wkb" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04ab2f2845abe967a213998446345a1a6fc9a8f318cd88f668a8e507861d13bc" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "writeable" version = "0.6.4" diff --git a/Cargo.toml b/Cargo.toml index 2ef245005..48191eeed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,11 @@ resolver = "3" datafusion = { version = "55", default-features = false } datafusion-proto = { version = "55", default-features = false } +# datafusion-iceberg depends on apache/iceberg-rust; use the same source so FileScanTask is one type. +[patch."https://github.com/apache/iceberg-rust"] +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } + [workspace.lints.clippy] disallowed_types = "deny" disallowed-methods = "deny" diff --git a/datafusion/distributed-iceberg-test-tools/Cargo.toml b/datafusion/distributed-iceberg-test-tools/Cargo.toml index c39d7ae69..5fa441099 100644 --- a/datafusion/distributed-iceberg-test-tools/Cargo.toml +++ b/datafusion/distributed-iceberg-test-tools/Cargo.toml @@ -14,8 +14,8 @@ datafusion = { workspace = true, features = ["parquet", "sql"] } # The test harness runs distributed queries on in-memory workers. datafusion-distributed = { path = "../distributed", features = ["integration"] } datafusion-proto = { workspace = true } -# Match datafusion-distributed-iceberg's iceberg-rust revision so tables share one type. -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +# Match datafusion-distributed-iceberg's iceberg-rust source so tables share one type. +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } rand = { version = "0.10", default-features = false, features = ["alloc"] } rand_pcg = "0.10" tempfile = "3.18" diff --git a/datafusion/distributed-iceberg/Cargo.toml b/datafusion/distributed-iceberg/Cargo.toml index 31e11fe37..7b355e1e3 100644 --- a/datafusion/distributed-iceberg/Cargo.toml +++ b/datafusion/distributed-iceberg/Cargo.toml @@ -12,11 +12,13 @@ async-trait = "0.1" bytes = "1" datafusion = { workspace = true, features = ["sql"] } datafusion-proto = { workspace = true } -# Match datafusion-iceberg's iceberg-rust revision so FileScanTask has one shared type. +# DataDog's iceberg-rust build of upstream main (Arrow 59), which exposes per-file column statistics on +# FileScanTask (apache/iceberg-rust#3424). The workspace [patch] makes datafusion-iceberg use the same +# source so FileScanTask is one shared type. iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg.git", rev = "20327db58c7a44330366da00a34a26e866d0da4d" } futures = "0.3" -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } -iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } datafusion-distributed = { path = "../distributed", features = ["sql"] } delegate = "0.13" prost = "0.14.1" diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index 349a0fc33..a4b7c84f0 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -109,8 +109,14 @@ Snapshot distinct-count estimates are retained as an inexact fallback, capped by selected-file row counts when known. They are not recomputed for the selected files and can overestimate the filtered result. Sum statistics remain unknown. -Another manifest pass is needed because the pinned Iceberg API does not retain -column metrics in file tasks. Scan task counts continue to use whole-file bytes. +The metrics come from the planned file tasks themselves: the scan asks Iceberg to +retain the statistics columns' metrics (`TableScanBuilder::with_column_stats`), so no +manifest is read a second time. Scan task counts continue to use whole-file bytes. + +The retained metrics stay on each task, so they are also serialized to workers with the +file list (a few small per-column maps per file) and counted by the planning memory +reservation. iceberg-rust has no way to drop them from a planned task, which would +avoid that cost. The Datadog integration also exports `datafusion_distributed::ColumnByteStatisticsProvider`. To preserve available column-byte estimates through grouping keys and inner joins, diff --git a/datafusion/distributed-iceberg/src/config.rs b/datafusion/distributed-iceberg/src/config.rs index 35aac75fa..762088ff2 100644 --- a/datafusion/distributed-iceberg/src/config.rs +++ b/datafusion/distributed-iceberg/src/config.rs @@ -14,7 +14,7 @@ extensions_options! { pub row_group_filtering_enabled: bool, default = true /// Whether to apply row-level selections while reading Parquet files. pub row_selection_enabled: bool, default = false - /// Read column statistics during planning (may require another manifest pass). + /// Read column statistics during planning (retained from the files selected during planning). /// Eager scans aggregate selected-file metrics and retain inexact snapshot NDV. pub column_stats_enabled: bool, default = false /// Discover and retain file tasks during physical planning instead of execution. diff --git a/datafusion/distributed-iceberg/src/eager/planned_files.rs b/datafusion/distributed-iceberg/src/eager/planned_files.rs index eb7f7c9cf..d9f188b01 100644 --- a/datafusion/distributed-iceberg/src/eager/planned_files.rs +++ b/datafusion/distributed-iceberg/src/eager/planned_files.rs @@ -6,7 +6,8 @@ use datafusion::common::{Result, Statistics}; use datafusion::execution::TaskContext; use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation}; use futures::TryStreamExt; -use iceberg::scan::{FileScanTask, TableScan}; +use iceberg::scan::{FileScanTask, FileScanTaskColumnStats, TableScan}; +use iceberg::spec::{Datum, PrimitiveLiteral}; use crate::common::df_err; @@ -86,6 +87,9 @@ fn retained_size(task: &FileScanTask) -> usize { + size_of_val(task.project_field_ids()) + size_of_val(task.deletes()) + task.key_metadata().map_or(0, <[u8]>::len); + if let Some(stats) = task.column_stats() { + bytes = bytes.saturating_add(column_stats_size(stats)); + } for delete in task.deletes() { bytes = bytes .saturating_add(delete.file_path().len()) @@ -95,3 +99,29 @@ fn retained_size(task: &FileScanTask) -> usize { } bytes } + +/// Approximate heap size of the metrics a task retains for column statistics. +fn column_stats_size(stats: &FileScanTaskColumnStats) -> usize { + let counts = [ + stats.column_sizes(), + stats.value_counts(), + stats.null_value_counts(), + stats.nan_value_counts(), + ] + .iter() + .map(|counts| counts.len() * size_of::<(i32, u64)>()) + .sum::(); + let bounds = [stats.lower_bounds(), stats.upper_bounds()] + .iter() + .flat_map(|bounds| bounds.values()) + .map(|bound| { + size_of::<(i32, Datum)>() + + match bound.literal() { + PrimitiveLiteral::String(value) => value.len(), + PrimitiveLiteral::Binary(value) => value.len(), + _ => 0, + } + }) + .sum::(); + size_of::() + counts + bounds +} diff --git a/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs b/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs index 2e51afe54..fc5b97de1 100644 --- a/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs +++ b/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs @@ -1,23 +1,48 @@ use crate::common::df_err; use datafusion::common::runtime::JoinSet; use datafusion::common::stats::Precision; -use datafusion::common::{ColumnStatistics, HashMap, HashSet, Result, exec_err}; +use datafusion::common::{ColumnStatistics, HashMap, Result, exec_err}; use datafusion::error::DataFusionError; use datafusion::scalar::ScalarValue; use iceberg::puffin::APACHE_DATASKETCHES_THETA_V1; use iceberg::spec::{ DataFile, Datum, ManifestContentType, ManifestList, NestedFieldRef, PrimitiveLiteral, - PrimitiveType, SnapshotRef, Type, + PrimitiveType, Schema, SnapshotRef, Type, }; use iceberg::table::Table; use std::sync::Arc; -use iceberg::scan::FileScanTask; +use iceberg::scan::{FileScanTask, FileScanTaskColumnStats}; + +/// Names of the primitive leaf columns under `fields`, in schema-dotted form. Passed to +/// `TableScanBuilder::with_column_stats` so each planned `FileScanTask` carries the file's +/// metrics for exactly the columns these statistics cover. Leaves that are not primitive +/// (variants) have no file metrics and are skipped. +pub(crate) fn column_stats_names( + schema: &Schema, + fields: &[NestedFieldRef], +) -> Result> { + let mut names = vec![]; + for field in fields { + let mut leaves = vec![]; + leaves_of(field, &mut leaves); + for leaf in leaves { + if !matches!(leaf.field_type.as_ref(), Type::Primitive(_)) { + continue; + } + let Some(name) = schema.name_by_field_id(leaf.id) else { + return exec_err!("Iceberg column {} has no name in the scan schema", leaf.id); + }; + names.push(name.to_string()); + } + } + Ok(names) +} /// Selected-file metrics, with snapshot NDV retained only as an inexact fallback. -/// FileScanTask does not retain manifest metrics, so this requires a metadata-only -/// second pass. Stop as soon as all selected files have been found. -pub(crate) async fn selected_column_statistics( +/// Metrics come from the `FileScanTask::column_stats` that the scan retained while +/// planning (see [`column_stats_names`]), so no manifest is read a second time. +pub(crate) fn selected_column_statistics( table: &Table, snapshot: &SnapshotRef, fields: &[NestedFieldRef], @@ -42,10 +67,6 @@ pub(crate) async fn selected_column_statistics( // Full-file metrics cannot describe a byte-range task. return Ok(vec![ColumnStatistics::new_unknown(); fields.len()]); } - let mut remaining: HashSet<_> = files.iter().map(|task| task.data_file_path()).collect(); - if remaining.len() != files.len() { - return exec_err!("Duplicate whole-file tasks in Iceberg column statistics lookup"); - } let field_ids: Vec<_> = fields .iter() .map(|field| { @@ -55,51 +76,31 @@ pub(crate) async fn selected_column_statistics( }) .collect(); let mut statistics = vec![None; fields.len()]; - let manifests = table - .manifest_list_reader(snapshot) - .load() - .await - .map_err(df_err)?; - for manifest in manifests - .entries() - .iter() - .filter(|manifest| manifest.content == ManifestContentType::Data) - { - let manifest = table - .manifest_reader() - .read(manifest) - .await - .map_err(df_err)?; - for entry in manifest.entries().iter().filter(|entry| entry.is_alive()) { - let file = entry.data_file(); - if !remaining.remove(file.file_path()) { - continue; - } - for ((acc, field), ids) in statistics.iter_mut().zip(fields).zip(&field_ids) { - // Leaf metrics cannot describe a parent struct/list/map's value or null count. - let mut next = if matches!(field.field_type.as_ref(), Type::Primitive(_)) { - data_file_col_stats(file, field.id) - } else { - ColumnStatistics::new_unknown() - }; - next.byte_size = ids - .iter() - .try_fold(0_usize, |total, id| { - let size = usize::try_from(*file.column_sizes().get(id)?).ok()?; - total.checked_add(size) - }) - .map(Precision::Inexact) - .unwrap_or(Precision::Absent); - *acc = Some(merge_col_stats(acc.take(), next)); - } - } - if remaining.is_empty() { - break; + for task in files { + let Some(file) = task.column_stats() else { + return exec_err!( + "Planned Iceberg file {} carries no column statistics", + task.data_file_path() + ); + }; + for ((acc, field), ids) in statistics.iter_mut().zip(fields).zip(&field_ids) { + // Leaf metrics cannot describe a parent struct/list/map's value or null count. + let mut next = if matches!(field.field_type.as_ref(), Type::Primitive(_)) { + task_col_stats(file, field.id) + } else { + ColumnStatistics::new_unknown() + }; + next.byte_size = ids + .iter() + .try_fold(0_usize, |total, id| { + let size = usize::try_from(*file.column_sizes().get(id)?).ok()?; + total.checked_add(size) + }) + .map(Precision::Inexact) + .unwrap_or(Precision::Absent); + *acc = Some(merge_col_stats(acc.take(), next)); } } - if !remaining.is_empty() { - return exec_err!("Planned Iceberg files missing from snapshot manifests"); - } let rows = files.iter().try_fold(0_usize, |rows, task| { rows.checked_add(usize::try_from(task.record_count()?).ok()?) }); @@ -123,17 +124,23 @@ pub(crate) async fn selected_column_statistics( } fn leaf_ids(field: &NestedFieldRef, ids: &mut Vec) { + let mut leaves = vec![]; + leaves_of(field, &mut leaves); + ids.extend(leaves.iter().map(|leaf| leaf.id)); +} + +fn leaves_of<'a>(field: &'a NestedFieldRef, leaves: &mut Vec<&'a NestedFieldRef>) { match field.field_type.as_ref() { - Type::Primitive(_) | Type::Variant(_) => ids.push(field.id), + Type::Primitive(_) | Type::Variant(_) => leaves.push(field), Type::Struct(fields) => { for field in fields.fields() { - leaf_ids(field, ids); + leaves_of(field, leaves); } } - Type::List(list) => leaf_ids(&list.element_field, ids), + Type::List(list) => leaves_of(&list.element_field, leaves), Type::Map(map) => { - leaf_ids(&map.key_field, ids); - leaf_ids(&map.value_field, ids); + leaves_of(&map.key_field, leaves); + leaves_of(&map.value_field, leaves); } } } @@ -279,28 +286,43 @@ fn merge_col_stats(acc: Option, next: ColumnStatistics) -> Col } fn data_file_col_stats(df: &DataFile, id: i32) -> ColumnStatistics { + file_col_stats( + df.null_value_counts().get(&id).copied(), + df.lower_bounds().get(&id), + df.upper_bounds().get(&id), + df.column_sizes().get(&id).copied(), + ) +} + +fn task_col_stats(stats: &FileScanTaskColumnStats, id: i32) -> ColumnStatistics { + file_col_stats( + stats.null_value_counts().get(&id).copied(), + stats.lower_bounds().get(&id), + stats.upper_bounds().get(&id), + stats.column_sizes().get(&id).copied(), + ) +} + +fn file_col_stats( + null_count: Option, + lower_bound: Option<&Datum>, + upper_bound: Option<&Datum>, + column_size: Option, +) -> ColumnStatistics { ColumnStatistics { - null_count: df - .null_value_counts() - .get(&id) - .map(|n| Precision::Exact(*n as usize)) + null_count: null_count + .map(|n| Precision::Exact(n as usize)) .unwrap_or(Precision::Absent), - min_value: df - .lower_bounds() - .get(&id) + min_value: lower_bound .and_then(datum_to_scalar) .map(Precision::Inexact) .unwrap_or(Precision::Absent), - max_value: df - .upper_bounds() - .get(&id) + max_value: upper_bound .and_then(datum_to_scalar) .map(Precision::Inexact) .unwrap_or(Precision::Absent), - byte_size: df - .column_sizes() - .get(&id) - .map(|n| Precision::Inexact(*n as usize)) + byte_size: column_size + .map(|n| Precision::Inexact(n as usize)) .unwrap_or(Precision::Absent), sum_value: Precision::Absent, distinct_count: Precision::Absent, diff --git a/datafusion/distributed-iceberg/src/table_provider/scan.rs b/datafusion/distributed-iceberg/src/table_provider/scan.rs index 8a70b4e93..943332c77 100644 --- a/datafusion/distributed-iceberg/src/table_provider/scan.rs +++ b/datafusion/distributed-iceberg/src/table_provider/scan.rs @@ -2,7 +2,9 @@ use crate::common::df_err; use crate::common::scan_config::{ScanConfig, apply_column_statistics}; use crate::eager::assign_files; use crate::eager::planned_files::PlannedFiles; -use crate::table_provider::column_statistics::{compute_column_stats, selected_column_statistics}; +use crate::table_provider::column_statistics::{ + column_stats_names, compute_column_stats, selected_column_statistics, +}; use crate::{EagerIcebergDataSource, IcebergConfig, IcebergWorkUnitFeed, LazyIcebergDataSource}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{Result, config_err, exec_err}; @@ -60,6 +62,25 @@ pub(crate) async fn plan_scan( .unwrap_or_else(iceberg::Runtime::current), &context, ); + // The columns whose statistics are requested, with the schema that names them. + let stats_fields = if IcebergConfig::from_task_context(&context).column_stats_enabled { + let schema = match &table_snapshot { + Some(snap) => snap.schema(table.metadata()).map_err(df_err)?, + // empty table + None => table.metadata().current_schema().clone(), + }; + let fields = schema.as_struct().fields(); + let fields: Vec<_> = match opts.projection { + Some(projection) => projection + .iter() + .map(|&idx| Arc::clone(&fields[idx])) + .collect(), + None => fields.to_vec(), + }; + Some((schema, fields)) + } else { + None + }; let planned = if IcebergConfig::from_task_context(&context).eager_file_planning { if partitioning.partition_count() == 0 { return config_err!("Iceberg scan requires at least one output partition"); @@ -74,6 +95,10 @@ pub(crate) async fn plan_scan( if let Some(predicates) = &predicates { builder = builder.with_filter(predicates.clone()); } + if let Some((schema, fields)) = &stats_fields { + // Keep each file's metrics on its task, so statistics need no second manifest pass. + builder = builder.with_column_stats(column_stats_names(schema, fields)?); + } let table_scan = builder.build().map_err(df_err)?; // File discovery runs on Iceberg's I/O runtime and is canceled if planning is dropped. let (_cancel_on_drop, canceled) = oneshot::channel::<()>(); @@ -84,7 +109,7 @@ pub(crate) async fn plan_scan( planned = PlannedFiles::collect(table_scan, &planning_context) => planned, } }); - Some(Arc::new(task.await.map_err(df_err)??)) + Some(task.await.map_err(df_err)??) } else { None }; @@ -92,33 +117,17 @@ pub(crate) async fn plan_scan( || planned .as_ref() .is_some_and(|files| files.tasks().iter().any(|task| !task.deletes().is_empty())); - let column_stats = if IcebergConfig::from_task_context(&context).column_stats_enabled { - let schema = match &table_snapshot { - Some(snap) => snap.schema(table.metadata()).map_err(df_err)?, - // empty table - None => table.metadata().current_schema().clone(), - }; - let fields = schema.as_struct().fields(); - let fields: Vec<_> = match opts.projection { - Some(projection) => projection - .iter() - .map(|&idx| Arc::clone(&fields[idx])) - .collect(), - None => fields.to_vec(), - }; + let column_stats = if let Some((_, fields)) = &stats_fields { if let Some(files) = &planned && let Some(snapshot) = &table_snapshot { - Some( - selected_column_statistics( - &table, - snapshot, - &fields, - files.tasks(), - may_filter_rows, - ) - .await?, - ) + Some(selected_column_statistics( + &table, + snapshot, + fields, + files.tasks(), + may_filter_rows, + )?) } else { let field_ids = fields.iter().map(|field| field.id).collect(); Some(compute_column_stats(table.clone(), field_ids, table_snapshot.clone()).await?) @@ -127,6 +136,7 @@ pub(crate) async fn plan_scan( None }; if let Some(planned) = planned { + let planned = Arc::new(planned); let mut statistics = planned.statistics(&scan.schema, may_filter_rows); if let Some(column_stats) = column_stats { apply_column_statistics(&mut statistics, column_stats); From 52556a2c040341aa1d8481de1cec77a036494c26 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Sat, 21 Nov 2026 02:22:58 +0000 Subject: [PATCH 4/4] Pin Iceberg to the Arrow 59 revision from dd-source#196245 Signed-off-by: alex.bianchi --- Cargo.lock | 7 ++++--- Cargo.toml | 4 ++-- datafusion/distributed-iceberg-test-tools/Cargo.toml | 2 +- datafusion/distributed-iceberg/Cargo.toml | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cac5ef273..f1f9f42ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4064,7 +4064,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" +source = "git+https://github.com/DataDog/iceberg-rust?rev=7fda030197a61057422b12ca4cf2cc783a5607f0#7fda030197a61057422b12ca4cf2cc783a5607f0" dependencies = [ "aes-gcm", "anyhow", @@ -4091,6 +4091,7 @@ dependencies = [ "fastnum", "flate2", "fnv", + "foldhash 0.2.0", "form_urlencoded", "futures", "iceberg-property-macro", @@ -4142,7 +4143,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.1" -source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" +source = "git+https://github.com/DataDog/iceberg-rust?rev=7fda030197a61057422b12ca4cf2cc783a5607f0#7fda030197a61057422b12ca4cf2cc783a5607f0" dependencies = [ "proc-macro2", "quote", @@ -4152,7 +4153,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/DataDog/iceberg-rust?rev=9354b42ad8c28b9f3df2ae4c3e4931beab0be12b#9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" +source = "git+https://github.com/DataDog/iceberg-rust?rev=7fda030197a61057422b12ca4cf2cc783a5607f0#7fda030197a61057422b12ca4cf2cc783a5607f0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 48191eeed..df3731c17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,8 +20,8 @@ datafusion-proto = { version = "55", default-features = false } # datafusion-iceberg depends on apache/iceberg-rust; use the same source so FileScanTask is one type. [patch."https://github.com/apache/iceberg-rust"] -iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } -iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "7fda030197a61057422b12ca4cf2cc783a5607f0" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "7fda030197a61057422b12ca4cf2cc783a5607f0" } [workspace.lints.clippy] disallowed_types = "deny" diff --git a/datafusion/distributed-iceberg-test-tools/Cargo.toml b/datafusion/distributed-iceberg-test-tools/Cargo.toml index 5fa441099..3517eb5c3 100644 --- a/datafusion/distributed-iceberg-test-tools/Cargo.toml +++ b/datafusion/distributed-iceberg-test-tools/Cargo.toml @@ -15,7 +15,7 @@ datafusion = { workspace = true, features = ["parquet", "sql"] } datafusion-distributed = { path = "../distributed", features = ["integration"] } datafusion-proto = { workspace = true } # Match datafusion-distributed-iceberg's iceberg-rust source so tables share one type. -iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "7fda030197a61057422b12ca4cf2cc783a5607f0" } rand = { version = "0.10", default-features = false, features = ["alloc"] } rand_pcg = "0.10" tempfile = "3.18" diff --git a/datafusion/distributed-iceberg/Cargo.toml b/datafusion/distributed-iceberg/Cargo.toml index 7b355e1e3..39be09a2e 100644 --- a/datafusion/distributed-iceberg/Cargo.toml +++ b/datafusion/distributed-iceberg/Cargo.toml @@ -17,8 +17,8 @@ datafusion-proto = { workspace = true } # source so FileScanTask is one shared type. iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg.git", rev = "20327db58c7a44330366da00a34a26e866d0da4d" } futures = "0.3" -iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b" } -iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "9354b42ad8c28b9f3df2ae4c3e4931beab0be12b", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "7fda030197a61057422b12ca4cf2cc783a5607f0" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "7fda030197a61057422b12ca4cf2cc783a5607f0", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } datafusion-distributed = { path = "../distributed", features = ["sql"] } delegate = "0.13" prost = "0.14.1"