From e59ba980553f4fec68d9c6a76aa0cd59b5461ab3 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Thu, 19 Nov 2026 20:06:51 -0500 Subject: [PATCH 1/5] feat: derive eager scan statistics from selected files Adapt selected-file statistics from datafusion-contrib/datafusion-distributed#787 while retaining conservative row-count precision and projected output-byte estimates. Signed-off-by: alex.bianchi --- .../src/read/file_scan_task_provider/eager.rs | 91 ++++++++++++++++--- .../src/read/file_scan_task_provider/lazy.rs | 5 +- .../tests/file_scan_task_provider.rs | 17 ++-- crates/datafusion/tests/statistics.rs | 37 +++++++- 4 files changed, 122 insertions(+), 28 deletions(-) diff --git a/crates/datafusion/src/read/file_scan_task_provider/eager.rs b/crates/datafusion/src/read/file_scan_task_provider/eager.rs index 8421a6e..e643d23 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/eager.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/eager.rs @@ -16,6 +16,7 @@ // under the License. use datafusion::arrow::datatypes::Schema; +use datafusion::common::stats::Precision; use datafusion::common::{Result, Statistics, exec_err}; use datafusion::physical_plan::Partitioning; use futures::StreamExt; @@ -28,12 +29,11 @@ use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext}; /// Files discovered before execution, in groups that each become one output /// partition. Group order and empty groups are retained. +/// Row estimates use the retained file record counts before predicates and +/// deletes. Only an empty assignment reports an exact row count. #[derive(Debug)] pub struct EagerFileScanTaskProvider { file_scan_tasks: Vec>, - /// Whole-assignment statistics retained when converting lazy discovery; - /// external assignments have no retained estimates. - pub(super) statistics: Option, } impl EagerFileScanTaskProvider { @@ -48,10 +48,7 @@ impl EagerFileScanTaskProvider { "An Iceberg task provider requires at least one file group" ); } - Ok(Self { - file_scan_tasks, - statistics: None, - }) + Ok(Self { file_scan_tasks }) } /// All file tasks, in partition order and then group order. @@ -93,12 +90,28 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider { if partition.is_some_and(|partition| partition >= self.groups().len()) { return exec_err!("Invalid Iceberg scan partition {partition:?}"); } - if (partition.is_none() || self.groups().len() == 1) - && let Some(statistics) = &self.statistics - { - return Ok(statistics.clone()); + let groups = match partition { + Some(partition) => &self.groups()[partition..=partition], + None => self.groups(), + }; + let mut statistics = Statistics::new_unknown(schema); + if groups.iter().all(Vec::is_empty) { + statistics.num_rows = Precision::Exact(0); + statistics.total_byte_size = Precision::Exact(0); + } else { + // Record counts precede predicates and deletes. Keep them inexact even + // for unfiltered scans, so metadata cannot answer COUNT(*) directly. + statistics.num_rows = groups + .iter() + .flatten() + .try_fold(0_usize, |rows, task| { + rows.checked_add(usize::try_from(task.record_count()?).ok()?) + }) + .map(Precision::Inexact) + .unwrap_or(Precision::Absent); } - Ok(Statistics::new_unknown(schema)) + // The source estimates projected output bytes, not compressed file sizes. + Ok(statistics) } fn fmt_as(&self, f: &mut Formatter) -> std::fmt::Result { @@ -124,3 +137,57 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider { Ok(()) } } + +#[cfg(test)] +mod tests { + use iceberg::spec::{DataFileFormat, Schema as IcebergSchema}; + + use super::*; + + #[test] + fn leaves_rows_unknown_when_a_task_has_no_record_count() -> Result<()> { + let provider = EagerFileScanTaskProvider::try_new(vec![vec![ + file_task(Some(10)), + file_task(None), + ]])?; + let statistics = provider.partition_statistics(None, &Schema::empty())?; + assert_eq!(statistics.num_rows, Precision::Absent); + Ok(()) + } + + #[test] + fn leaves_rows_unknown_when_record_counts_overflow() -> Result<()> { + let provider = EagerFileScanTaskProvider::try_new(vec![ + vec![file_task(Some(u64::MAX))], + vec![file_task(Some(1))], + ])?; + let statistics = provider.partition_statistics(None, &Schema::empty())?; + assert_eq!(statistics.num_rows, Precision::Absent); + Ok(()) + } + + #[test] + fn keeps_zero_file_record_counts_inexact() -> Result<()> { + let provider = + EagerFileScanTaskProvider::try_new(vec![vec![file_task(Some(0))]])?; + let statistics = provider.partition_statistics(None, &Schema::empty())?; + assert_eq!(statistics.num_rows, Precision::Inexact(0)); + Ok(()) + } + + // Only metadata is inspected; these files are never opened. + fn file_task(record_count: Option) -> FileScanTask { + FileScanTask::builder() + .with_file_size_in_bytes(1) + .with_start(0) + .with_length(1) + .with_record_count(record_count) + .with_data_file_path("data.parquet".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(Arc::new(IcebergSchema::builder().build().unwrap())) + .with_project_field_ids(vec![]) + .with_case_sensitive(true) + .build() + .unwrap() + } +} diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index fdd6824..d483c41 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -62,7 +62,6 @@ impl LazyFileScanTaskProvider { "Iceberg eager planning requires at least one target partition" ); } - let statistics = self.partition_statistics(None, &self.schema)?; let mut tasks = Arc::new(self).open( 0, FileScanTaskProviderContext { @@ -81,9 +80,7 @@ impl LazyFileScanTaskProvider { if groups.is_empty() { groups.push(Vec::new()); } - let mut provider = EagerFileScanTaskProvider::try_new(groups)?; - provider.statistics = Some(statistics); - Ok(provider) + EagerFileScanTaskProvider::try_new(groups) } /// Table retained for discovery, including its metadata and storage access. diff --git a/crates/datafusion/tests/file_scan_task_provider.rs b/crates/datafusion/tests/file_scan_task_provider.rs index 659713b..253ef12 100644 --- a/crates/datafusion/tests/file_scan_task_provider.rs +++ b/crates/datafusion/tests/file_scan_task_provider.rs @@ -144,13 +144,8 @@ async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> { .await?; insta::assert_snapshot!(plan, @" - ProjectionExec: expr=[count(Int64(1))@0 as trips] - AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] - CoalescePartitionsExec - AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] - FilterExec: pickup_date@0 = 2025-01-01, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=eager, planned_files=0 + ProjectionExec: expr=[0 as trips] + PlaceholderRowExec "); insta::assert_snapshot!(rows, @" +-------+ @@ -196,7 +191,7 @@ async fn replacing_the_provider_preserves_the_fetch_limit() -> Result<()> { } #[tokio::test] -async fn eager_snapshot_statistics_apply_fetch_per_partition() -> Result<()> { +async fn eager_file_statistics_apply_fetch_per_partition() -> Result<()> { let fixtures = FixtureCatalog::try_new().await?; let schema = Schema::builder() .with_fields(vec![ @@ -231,12 +226,12 @@ async fn eager_snapshot_statistics_apply_fetch_per_partition() -> Result<()> { .build() .await?; - // Eager conversion retains the six-row snapshot estimate for the assignment. + // Each of the two file groups contributes three rows to the assignment. let plan = harness.physical_plan("SELECT id FROM items").await?; insta::assert_snapshot!(harness.source_statistics(&plan)?, @" all partitions (2): Rows=Inexact(6), Bytes=Inexact(48), [(Col[0]:)] - partition 0: Rows=Absent, Bytes=Absent, [(Col[0]:)] - partition 1: Rows=Absent, Bytes=Absent, [(Col[0]:)] + partition 0: Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]:)] + partition 1: Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]:)] "); // Fetch caps each source partition at two rows, not the whole assignment. diff --git a/crates/datafusion/tests/statistics.rs b/crates/datafusion/tests/statistics.rs index e45cda1..eafa31b 100644 --- a/crates/datafusion/tests/statistics.rs +++ b/crates/datafusion/tests/statistics.rs @@ -18,7 +18,7 @@ #[cfg(test)] mod tests { use datafusion::error::Result; - use iceberg_datafusion::IcebergTableProviderFactory; + use iceberg_datafusion::{IcebergDataFusionConfig, IcebergTableProviderFactory}; use iceberg_datafusion_test_tools::{IcebergTestHarness, create_taxi_table}; #[tokio::test] @@ -43,4 +43,39 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn eager_statistics_count_only_the_selected_files() -> Result<()> { + // Taxi has seven daily files of 25,000 rows; the predicate selects two. + let harness = IcebergTestHarness::builder() + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_table(create_taxi_table().await?) + .with_session_extension(|mut state| { + state + .config() + .get_or_insert_default() + .options_mut() + .extensions + .insert(IcebergDataFusionConfig::default()); + state + }) + .with_config_option("iceberg.execution.eager_file_planning", true) + .with_config_option("datafusion.execution.target_partitions", 2) + .build() + .await?; + + let plan = harness + .physical_plan( + "SELECT vendor_id FROM taxi \ + WHERE pickup_date IN (DATE '2024-01-10', DATE '2024-01-11')", + ) + .await?; + + insta::assert_snapshot!(harness.source_statistics(&plan)?, @" + all partitions (2): Rows=Inexact(50000), Bytes=Inexact(400000), [(Col[0]:),(Col[1]:)] + partition 0: Rows=Inexact(25000), Bytes=Inexact(200000), [(Col[0]:),(Col[1]:)] + partition 1: Rows=Inexact(25000), Bytes=Inexact(200000), [(Col[0]:),(Col[1]:)] + "); + Ok(()) + } } From 3ae2b3be9725270ee2d20c2c2aab0a4b39ccc20e Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Thu, 19 Nov 2026 20:17:51 -0500 Subject: [PATCH 2/5] fix: preserve exact empty scan statistics through fetch Signed-off-by: alex.bianchi --- crates/datafusion/src/read/scan.rs | 59 +++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index 52ead92..313a966 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -297,6 +297,15 @@ impl DataSource for IcebergDataSource { self.schema.fields().len() ); } + // Empty input and a zero fetch prove there is no output independently + // of estimates or how rows are distributed across partitions. + if statistics.num_rows == Precision::Exact(0) || self.fetch == Some(0) { + return Ok(Arc::new( + Statistics::new_unknown(&self.schema) + .with_num_rows(Precision::Exact(0)) + .with_total_byte_size(Precision::Exact(0)), + )); + } let fetch = if partition.is_none() && partitions > 1 { // Fetch is enforced independently in each partition. Without its row // distribution we can only bound the sum, even for an exact input. @@ -354,7 +363,7 @@ mod tests { use iceberg::{Runtime, TableIdent}; use super::IcebergDataSource; - use crate::read::FileScanTaskProviderBuilder; + use crate::read::{EagerFileScanTaskProvider, FileScanTaskProviderBuilder}; /// Returns a table with one snapshot per summary, oldest first, with ids /// counting up from 1. The last snapshot is the current one. @@ -523,6 +532,54 @@ mod tests { assert_eq!(statistics.total_byte_size, Precision::Inexact(80)); } + #[tokio::test] + async fn reports_exact_zero_rows_and_bytes_for_zero_fetch() { + let table = table_with_snapshot_summaries(&[&[]]); + let statistics = id_scan_statistics(table, None, Some(0)).await; + assert_eq!(statistics.num_rows, Precision::Exact(0)); + assert_eq!(statistics.total_byte_size, Precision::Exact(0)); + } + + #[tokio::test] + async fn uses_fetch_as_an_inexact_bound_when_rows_are_unknown() { + let table = table_with_snapshot_summaries(&[&[]]); + let statistics = id_scan_statistics(table, None, Some(10)).await; + assert_eq!(statistics.num_rows, Precision::Inexact(10)); + assert_eq!(statistics.total_byte_size, Precision::Inexact(80)); + } + + #[tokio::test] + async fn preserves_exact_empty_assignments_with_fetch_across_partitions() { + let table = table_with_snapshot_summaries(&[]); + let context = SessionContext::new(); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "name", + DataType::Utf8, + true, + )])); + let provider = EagerFileScanTaskProvider::try_new(vec![vec![], vec![]]).unwrap(); + let source = IcebergDataSource::builder( + &context.copied_config(), + schema, + table.file_io().clone(), + table.reader_builder(), + Arc::new(provider), + ) + .with_fetch(Some(10)) + .build() + .unwrap(); + + for partition in [None, Some(0), Some(1)] { + let statistics = source.partition_statistics(partition).unwrap(); + assert_eq!(statistics.num_rows, Precision::Exact(0), "{partition:?}"); + assert_eq!( + statistics.total_byte_size, + Precision::Exact(0), + "{partition:?}" + ); + } + } + #[tokio::test] async fn omits_the_byte_size_of_variable_width_columns() { let table = table_with_snapshot_summaries(&[&[("total-records", "100")]]); From 56c6dc60e2036787cdd55cba953c5b9a18713a4a Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Thu, 19 Nov 2026 20:31:46 -0500 Subject: [PATCH 3/5] test: reuse the harness for empty assignment statistics Signed-off-by: alex.bianchi --- crates/datafusion/src/read/scan.rs | 42 +------------------ .../tests/file_scan_task_provider.rs | 14 ++++++- 2 files changed, 14 insertions(+), 42 deletions(-) diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index 313a966..17798b1 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -363,7 +363,7 @@ mod tests { use iceberg::{Runtime, TableIdent}; use super::IcebergDataSource; - use crate::read::{EagerFileScanTaskProvider, FileScanTaskProviderBuilder}; + use crate::read::FileScanTaskProviderBuilder; /// Returns a table with one snapshot per summary, oldest first, with ids /// counting up from 1. The last snapshot is the current one. @@ -540,46 +540,6 @@ mod tests { assert_eq!(statistics.total_byte_size, Precision::Exact(0)); } - #[tokio::test] - async fn uses_fetch_as_an_inexact_bound_when_rows_are_unknown() { - let table = table_with_snapshot_summaries(&[&[]]); - let statistics = id_scan_statistics(table, None, Some(10)).await; - assert_eq!(statistics.num_rows, Precision::Inexact(10)); - assert_eq!(statistics.total_byte_size, Precision::Inexact(80)); - } - - #[tokio::test] - async fn preserves_exact_empty_assignments_with_fetch_across_partitions() { - let table = table_with_snapshot_summaries(&[]); - let context = SessionContext::new(); - let schema = Arc::new(ArrowSchema::new(vec![Field::new( - "name", - DataType::Utf8, - true, - )])); - let provider = EagerFileScanTaskProvider::try_new(vec![vec![], vec![]]).unwrap(); - let source = IcebergDataSource::builder( - &context.copied_config(), - schema, - table.file_io().clone(), - table.reader_builder(), - Arc::new(provider), - ) - .with_fetch(Some(10)) - .build() - .unwrap(); - - for partition in [None, Some(0), Some(1)] { - let statistics = source.partition_statistics(partition).unwrap(); - assert_eq!(statistics.num_rows, Precision::Exact(0), "{partition:?}"); - assert_eq!( - statistics.total_byte_size, - Precision::Exact(0), - "{partition:?}" - ); - } - } - #[tokio::test] async fn omits_the_byte_size_of_variable_width_columns() { let table = table_with_snapshot_summaries(&[&[("total-records", "100")]]); diff --git a/crates/datafusion/tests/file_scan_task_provider.rs b/crates/datafusion/tests/file_scan_task_provider.rs index 253ef12..2c11252 100644 --- a/crates/datafusion/tests/file_scan_task_provider.rs +++ b/crates/datafusion/tests/file_scan_task_provider.rs @@ -158,7 +158,7 @@ async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> { } #[tokio::test] -async fn replacing_the_provider_preserves_the_fetch_limit() -> Result<()> { +async fn replacing_the_provider_preserves_fetch_and_updates_statistics() -> Result<()> { let harness = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) @@ -187,6 +187,18 @@ async fn replacing_the_provider_preserves_the_fetch_limit() -> Result<()> { let assigned_plan = DataSourceExec::from_data_source(assigned); insta::assert_snapshot!(displayable(assigned_plan.as_ref()).indent(true).to_string(), @"DataSourceExec: format=iceberg, projection=[pickup_date], provider=eager, planned_files=1, fetch=2"); + + // Reassigning no files keeps the fetch, but proves every partition is empty. + let empty = source.with_file_scan_task_provider(Arc::new( + EagerFileScanTaskProvider::try_new(vec![vec![], vec![]])?, + ))?; + assert_eq!(empty.fetch(), Some(2)); + let empty_plan: Arc = DataSourceExec::from_data_source(empty); + insta::assert_snapshot!(harness.source_statistics(&empty_plan)?, @" + all partitions (2): Rows=Exact(0), Bytes=Exact(0), [(Col[0]:)] + partition 0: Rows=Exact(0), Bytes=Exact(0), [(Col[0]:)] + partition 1: Rows=Exact(0), Bytes=Exact(0), [(Col[0]:)] + "); Ok(()) } From 9203aeb10279a12ff7605beff0d84b671665d7c2 Mon Sep 17 00:00:00 2001 From: alexanderbianchi <75697973+alexanderbianchi@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:00:09 -0500 Subject: [PATCH 4/5] fix: report known zero file totals as exact in eager statistics Signed-off-by: alex.bianchi --- .../src/read/file_scan_task_provider/eager.rs | 49 ++++++++----------- crates/datafusion/src/read/scan.rs | 17 ------- .../tests/file_scan_task_provider.rs | 16 +----- 3 files changed, 22 insertions(+), 60 deletions(-) diff --git a/crates/datafusion/src/read/file_scan_task_provider/eager.rs b/crates/datafusion/src/read/file_scan_task_provider/eager.rs index e643d23..a5105d9 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/eager.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/eager.rs @@ -29,8 +29,7 @@ use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext}; /// Files discovered before execution, in groups that each become one output /// partition. Group order and empty groups are retained. -/// Row estimates use the retained file record counts before predicates and -/// deletes. Only an empty assignment reports an exact row count. +/// Row estimates sum the retained file record counts before predicates and deletes. #[derive(Debug)] pub struct EagerFileScanTaskProvider { file_scan_tasks: Vec>, @@ -94,24 +93,16 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider { Some(partition) => &self.groups()[partition..=partition], None => self.groups(), }; - let mut statistics = Statistics::new_unknown(schema); - if groups.iter().all(Vec::is_empty) { - statistics.num_rows = Precision::Exact(0); - statistics.total_byte_size = Precision::Exact(0); - } else { - // Record counts precede predicates and deletes. Keep them inexact even - // for unfiltered scans, so metadata cannot answer COUNT(*) directly. - statistics.num_rows = groups - .iter() - .flatten() - .try_fold(0_usize, |rows, task| { - rows.checked_add(usize::try_from(task.record_count()?).ok()?) - }) - .map(Precision::Inexact) - .unwrap_or(Precision::Absent); - } - // The source estimates projected output bytes, not compressed file sizes. - Ok(statistics) + let rows = groups.iter().flatten().try_fold(0_usize, |rows, task| { + rows.checked_add(usize::try_from(task.record_count()?).ok()?) + }); + // Predicates and deletes may reduce a positive count, but cannot add rows. + let num_rows = match rows { + Some(0) => Precision::Exact(0), + Some(rows) => Precision::Inexact(rows), + None => Precision::Absent, + }; + Ok(Statistics::new_unknown(schema).with_num_rows(num_rows)) } fn fmt_as(&self, f: &mut Formatter) -> std::fmt::Result { @@ -144,6 +135,15 @@ mod tests { use super::*; + #[test] + fn a_file_with_zero_records_has_exactly_zero_rows() -> Result<()> { + let provider = + EagerFileScanTaskProvider::try_new(vec![vec![file_task(Some(0))]])?; + let statistics = provider.partition_statistics(None, &Schema::empty())?; + assert_eq!(statistics.num_rows, Precision::Exact(0)); + Ok(()) + } + #[test] fn leaves_rows_unknown_when_a_task_has_no_record_count() -> Result<()> { let provider = EagerFileScanTaskProvider::try_new(vec![vec![ @@ -166,15 +166,6 @@ mod tests { Ok(()) } - #[test] - fn keeps_zero_file_record_counts_inexact() -> Result<()> { - let provider = - EagerFileScanTaskProvider::try_new(vec![vec![file_task(Some(0))]])?; - let statistics = provider.partition_statistics(None, &Schema::empty())?; - assert_eq!(statistics.num_rows, Precision::Inexact(0)); - Ok(()) - } - // Only metadata is inspected; these files are never opened. fn file_task(record_count: Option) -> FileScanTask { FileScanTask::builder() diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index 17798b1..52ead92 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -297,15 +297,6 @@ impl DataSource for IcebergDataSource { self.schema.fields().len() ); } - // Empty input and a zero fetch prove there is no output independently - // of estimates or how rows are distributed across partitions. - if statistics.num_rows == Precision::Exact(0) || self.fetch == Some(0) { - return Ok(Arc::new( - Statistics::new_unknown(&self.schema) - .with_num_rows(Precision::Exact(0)) - .with_total_byte_size(Precision::Exact(0)), - )); - } let fetch = if partition.is_none() && partitions > 1 { // Fetch is enforced independently in each partition. Without its row // distribution we can only bound the sum, even for an exact input. @@ -532,14 +523,6 @@ mod tests { assert_eq!(statistics.total_byte_size, Precision::Inexact(80)); } - #[tokio::test] - async fn reports_exact_zero_rows_and_bytes_for_zero_fetch() { - let table = table_with_snapshot_summaries(&[&[]]); - let statistics = id_scan_statistics(table, None, Some(0)).await; - assert_eq!(statistics.num_rows, Precision::Exact(0)); - assert_eq!(statistics.total_byte_size, Precision::Exact(0)); - } - #[tokio::test] async fn omits_the_byte_size_of_variable_width_columns() { let table = table_with_snapshot_summaries(&[&[("total-records", "100")]]); diff --git a/crates/datafusion/tests/file_scan_task_provider.rs b/crates/datafusion/tests/file_scan_task_provider.rs index 2c11252..c9c3447 100644 --- a/crates/datafusion/tests/file_scan_task_provider.rs +++ b/crates/datafusion/tests/file_scan_task_provider.rs @@ -121,7 +121,7 @@ async fn eager_scan_plans_only_the_matching_taxi_files() -> Result<()> { } #[tokio::test] -async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> { +async fn eager_count_skips_scan_when_no_files_match() -> Result<()> { let harness = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) @@ -158,7 +158,7 @@ async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> { } #[tokio::test] -async fn replacing_the_provider_preserves_fetch_and_updates_statistics() -> Result<()> { +async fn replacing_the_provider_preserves_the_fetch_limit() -> Result<()> { let harness = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) @@ -187,18 +187,6 @@ async fn replacing_the_provider_preserves_fetch_and_updates_statistics() -> Resu let assigned_plan = DataSourceExec::from_data_source(assigned); insta::assert_snapshot!(displayable(assigned_plan.as_ref()).indent(true).to_string(), @"DataSourceExec: format=iceberg, projection=[pickup_date], provider=eager, planned_files=1, fetch=2"); - - // Reassigning no files keeps the fetch, but proves every partition is empty. - let empty = source.with_file_scan_task_provider(Arc::new( - EagerFileScanTaskProvider::try_new(vec![vec![], vec![]])?, - ))?; - assert_eq!(empty.fetch(), Some(2)); - let empty_plan: Arc = DataSourceExec::from_data_source(empty); - insta::assert_snapshot!(harness.source_statistics(&empty_plan)?, @" - all partitions (2): Rows=Exact(0), Bytes=Exact(0), [(Col[0]:)] - partition 0: Rows=Exact(0), Bytes=Exact(0), [(Col[0]:)] - partition 1: Rows=Exact(0), Bytes=Exact(0), [(Col[0]:)] - "); Ok(()) } From 1d456a950ab04538b483b03441b1f8531c962479 Mon Sep 17 00:00:00 2001 From: alexanderbianchi <75697973+alexanderbianchi@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:23:56 -0500 Subject: [PATCH 5/5] test: remove synthetic record count overflow case Signed-off-by: alex.bianchi --- .../src/read/file_scan_task_provider/eager.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/crates/datafusion/src/read/file_scan_task_provider/eager.rs b/crates/datafusion/src/read/file_scan_task_provider/eager.rs index a5105d9..bbdd672 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/eager.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/eager.rs @@ -155,17 +155,6 @@ mod tests { Ok(()) } - #[test] - fn leaves_rows_unknown_when_record_counts_overflow() -> Result<()> { - let provider = EagerFileScanTaskProvider::try_new(vec![ - vec![file_task(Some(u64::MAX))], - vec![file_task(Some(1))], - ])?; - let statistics = provider.partition_statistics(None, &Schema::empty())?; - assert_eq!(statistics.num_rows, Precision::Absent); - Ok(()) - } - // Only metadata is inspected; these files are never opened. fn file_task(record_count: Option) -> FileScanTask { FileScanTask::builder()