diff --git a/crates/datafusion/src/read/file_scan_task_provider/eager.rs b/crates/datafusion/src/read/file_scan_task_provider/eager.rs index 8421a6e..bbdd672 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/eager.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/eager.rs @@ -16,6 +16,7 @@ // under the License. use datafusion::arrow::datatypes::Schema; +use datafusion::common::stats::Precision; use datafusion::common::{Result, Statistics, exec_err}; use datafusion::physical_plan::Partitioning; use futures::StreamExt; @@ -28,12 +29,10 @@ use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext}; /// Files discovered before execution, in groups that each become one output /// partition. Group order and empty groups are retained. +/// Row estimates sum the retained file record counts before predicates and deletes. #[derive(Debug)] pub struct EagerFileScanTaskProvider { file_scan_tasks: Vec>, - /// Whole-assignment statistics retained when converting lazy discovery; - /// external assignments have no retained estimates. - pub(super) statistics: Option, } impl EagerFileScanTaskProvider { @@ -48,10 +47,7 @@ impl EagerFileScanTaskProvider { "An Iceberg task provider requires at least one file group" ); } - Ok(Self { - file_scan_tasks, - statistics: None, - }) + Ok(Self { file_scan_tasks }) } /// All file tasks, in partition order and then group order. @@ -93,12 +89,20 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider { if partition.is_some_and(|partition| partition >= self.groups().len()) { return exec_err!("Invalid Iceberg scan partition {partition:?}"); } - if (partition.is_none() || self.groups().len() == 1) - && let Some(statistics) = &self.statistics - { - return Ok(statistics.clone()); - } - Ok(Statistics::new_unknown(schema)) + let groups = match partition { + Some(partition) => &self.groups()[partition..=partition], + None => self.groups(), + }; + let rows = groups.iter().flatten().try_fold(0_usize, |rows, task| { + rows.checked_add(usize::try_from(task.record_count()?).ok()?) + }); + // Predicates and deletes may reduce a positive count, but cannot add rows. + let num_rows = match rows { + Some(0) => Precision::Exact(0), + Some(rows) => Precision::Inexact(rows), + None => Precision::Absent, + }; + Ok(Statistics::new_unknown(schema).with_num_rows(num_rows)) } fn fmt_as(&self, f: &mut Formatter) -> std::fmt::Result { @@ -124,3 +128,46 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider { Ok(()) } } + +#[cfg(test)] +mod tests { + use iceberg::spec::{DataFileFormat, Schema as IcebergSchema}; + + use super::*; + + #[test] + fn a_file_with_zero_records_has_exactly_zero_rows() -> Result<()> { + let provider = + EagerFileScanTaskProvider::try_new(vec![vec![file_task(Some(0))]])?; + let statistics = provider.partition_statistics(None, &Schema::empty())?; + assert_eq!(statistics.num_rows, Precision::Exact(0)); + Ok(()) + } + + #[test] + fn leaves_rows_unknown_when_a_task_has_no_record_count() -> Result<()> { + let provider = EagerFileScanTaskProvider::try_new(vec![vec![ + file_task(Some(10)), + file_task(None), + ]])?; + let statistics = provider.partition_statistics(None, &Schema::empty())?; + assert_eq!(statistics.num_rows, Precision::Absent); + Ok(()) + } + + // Only metadata is inspected; these files are never opened. + fn file_task(record_count: Option) -> FileScanTask { + FileScanTask::builder() + .with_file_size_in_bytes(1) + .with_start(0) + .with_length(1) + .with_record_count(record_count) + .with_data_file_path("data.parquet".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(Arc::new(IcebergSchema::builder().build().unwrap())) + .with_project_field_ids(vec![]) + .with_case_sensitive(true) + .build() + .unwrap() + } +} diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index fdd6824..d483c41 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -62,7 +62,6 @@ impl LazyFileScanTaskProvider { "Iceberg eager planning requires at least one target partition" ); } - let statistics = self.partition_statistics(None, &self.schema)?; let mut tasks = Arc::new(self).open( 0, FileScanTaskProviderContext { @@ -81,9 +80,7 @@ impl LazyFileScanTaskProvider { if groups.is_empty() { groups.push(Vec::new()); } - let mut provider = EagerFileScanTaskProvider::try_new(groups)?; - provider.statistics = Some(statistics); - Ok(provider) + EagerFileScanTaskProvider::try_new(groups) } /// Table retained for discovery, including its metadata and storage access. diff --git a/crates/datafusion/tests/file_scan_task_provider.rs b/crates/datafusion/tests/file_scan_task_provider.rs index 659713b..c9c3447 100644 --- a/crates/datafusion/tests/file_scan_task_provider.rs +++ b/crates/datafusion/tests/file_scan_task_provider.rs @@ -121,7 +121,7 @@ async fn eager_scan_plans_only_the_matching_taxi_files() -> Result<()> { } #[tokio::test] -async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> { +async fn eager_count_skips_scan_when_no_files_match() -> Result<()> { let harness = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) @@ -144,13 +144,8 @@ async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> { .await?; insta::assert_snapshot!(plan, @" - ProjectionExec: expr=[count(Int64(1))@0 as trips] - AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] - CoalescePartitionsExec - AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] - FilterExec: pickup_date@0 = 2025-01-01, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=eager, planned_files=0 + ProjectionExec: expr=[0 as trips] + PlaceholderRowExec "); insta::assert_snapshot!(rows, @" +-------+ @@ -196,7 +191,7 @@ async fn replacing_the_provider_preserves_the_fetch_limit() -> Result<()> { } #[tokio::test] -async fn eager_snapshot_statistics_apply_fetch_per_partition() -> Result<()> { +async fn eager_file_statistics_apply_fetch_per_partition() -> Result<()> { let fixtures = FixtureCatalog::try_new().await?; let schema = Schema::builder() .with_fields(vec![ @@ -231,12 +226,12 @@ async fn eager_snapshot_statistics_apply_fetch_per_partition() -> Result<()> { .build() .await?; - // Eager conversion retains the six-row snapshot estimate for the assignment. + // Each of the two file groups contributes three rows to the assignment. let plan = harness.physical_plan("SELECT id FROM items").await?; insta::assert_snapshot!(harness.source_statistics(&plan)?, @" all partitions (2): Rows=Inexact(6), Bytes=Inexact(48), [(Col[0]:)] - partition 0: Rows=Absent, Bytes=Absent, [(Col[0]:)] - partition 1: Rows=Absent, Bytes=Absent, [(Col[0]:)] + partition 0: Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]:)] + partition 1: Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]:)] "); // Fetch caps each source partition at two rows, not the whole assignment. diff --git a/crates/datafusion/tests/statistics.rs b/crates/datafusion/tests/statistics.rs index e45cda1..eafa31b 100644 --- a/crates/datafusion/tests/statistics.rs +++ b/crates/datafusion/tests/statistics.rs @@ -18,7 +18,7 @@ #[cfg(test)] mod tests { use datafusion::error::Result; - use iceberg_datafusion::IcebergTableProviderFactory; + use iceberg_datafusion::{IcebergDataFusionConfig, IcebergTableProviderFactory}; use iceberg_datafusion_test_tools::{IcebergTestHarness, create_taxi_table}; #[tokio::test] @@ -43,4 +43,39 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn eager_statistics_count_only_the_selected_files() -> Result<()> { + // Taxi has seven daily files of 25,000 rows; the predicate selects two. + let harness = IcebergTestHarness::builder() + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_table(create_taxi_table().await?) + .with_session_extension(|mut state| { + state + .config() + .get_or_insert_default() + .options_mut() + .extensions + .insert(IcebergDataFusionConfig::default()); + state + }) + .with_config_option("iceberg.execution.eager_file_planning", true) + .with_config_option("datafusion.execution.target_partitions", 2) + .build() + .await?; + + let plan = harness + .physical_plan( + "SELECT vendor_id FROM taxi \ + WHERE pickup_date IN (DATE '2024-01-10', DATE '2024-01-11')", + ) + .await?; + + insta::assert_snapshot!(harness.source_statistics(&plan)?, @" + all partitions (2): Rows=Inexact(50000), Bytes=Inexact(400000), [(Col[0]:),(Col[1]:)] + partition 0: Rows=Inexact(25000), Bytes=Inexact(200000), [(Col[0]:),(Col[1]:)] + partition 1: Rows=Inexact(25000), Bytes=Inexact(200000), [(Col[0]:),(Col[1]:)] + "); + Ok(()) + } }