Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 60 additions & 13 deletions crates/datafusion/src/read/file_scan_task_provider/eager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
// under the License.

use datafusion::arrow::datatypes::Schema;
use datafusion::common::stats::Precision;
use datafusion::common::{Result, Statistics, exec_err};
use datafusion::physical_plan::Partitioning;
use futures::StreamExt;
Expand All @@ -28,12 +29,10 @@ use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext};

/// Files discovered before execution, in groups that each become one output
/// partition. Group order and empty groups are retained.
/// Row estimates sum the retained file record counts before predicates and deletes.
#[derive(Debug)]
pub struct EagerFileScanTaskProvider {
file_scan_tasks: Vec<Vec<FileScanTask>>,
/// Whole-assignment statistics retained when converting lazy discovery;
/// external assignments have no retained estimates.
pub(super) statistics: Option<Statistics>,
}

impl EagerFileScanTaskProvider {
Expand All @@ -48,10 +47,7 @@ impl EagerFileScanTaskProvider {
"An Iceberg task provider requires at least one file group"
);
}
Ok(Self {
file_scan_tasks,
statistics: None,
})
Ok(Self { file_scan_tasks })
}

/// All file tasks, in partition order and then group order.
Expand Down Expand Up @@ -93,12 +89,20 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider {
if partition.is_some_and(|partition| partition >= self.groups().len()) {
return exec_err!("Invalid Iceberg scan partition {partition:?}");
}
if (partition.is_none() || self.groups().len() == 1)
&& let Some(statistics) = &self.statistics
{
return Ok(statistics.clone());
}
Ok(Statistics::new_unknown(schema))
let groups = match partition {
Some(partition) => &self.groups()[partition..=partition],
None => self.groups(),
};
let rows = groups.iter().flatten().try_fold(0_usize, |rows, task| {
rows.checked_add(usize::try_from(task.record_count()?).ok()?)
});
// Predicates and deletes may reduce a positive count, but cannot add rows.
let num_rows = match rows {
Some(0) => Precision::Exact(0),
Some(rows) => Precision::Inexact(rows),
None => Precision::Absent,
};
Ok(Statistics::new_unknown(schema).with_num_rows(num_rows))
}

fn fmt_as(&self, f: &mut Formatter) -> std::fmt::Result {
Expand All @@ -124,3 +128,46 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider {
Ok(())
}
}

#[cfg(test)]
mod tests {
use iceberg::spec::{DataFileFormat, Schema as IcebergSchema};

use super::*;

#[test]
fn a_file_with_zero_records_has_exactly_zero_rows() -> Result<()> {
let provider =
EagerFileScanTaskProvider::try_new(vec![vec![file_task(Some(0))]])?;
let statistics = provider.partition_statistics(None, &Schema::empty())?;
assert_eq!(statistics.num_rows, Precision::Exact(0));
Ok(())
}

#[test]
fn leaves_rows_unknown_when_a_task_has_no_record_count() -> Result<()> {
let provider = EagerFileScanTaskProvider::try_new(vec![vec![
file_task(Some(10)),
file_task(None),
]])?;
let statistics = provider.partition_statistics(None, &Schema::empty())?;
assert_eq!(statistics.num_rows, Precision::Absent);
Ok(())
}

// Only metadata is inspected; these files are never opened.
fn file_task(record_count: Option<u64>) -> FileScanTask {
FileScanTask::builder()
.with_file_size_in_bytes(1)
.with_start(0)
.with_length(1)
.with_record_count(record_count)
.with_data_file_path("data.parquet".to_string())
.with_data_file_format(DataFileFormat::Parquet)
.with_schema(Arc::new(IcebergSchema::builder().build().unwrap()))
.with_project_field_ids(vec![])
.with_case_sensitive(true)
.build()
.unwrap()
}
}
5 changes: 1 addition & 4 deletions crates/datafusion/src/read/file_scan_task_provider/lazy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ impl LazyFileScanTaskProvider {
"Iceberg eager planning requires at least one target partition"
);
}
let statistics = self.partition_statistics(None, &self.schema)?;
let mut tasks = Arc::new(self).open(
0,
FileScanTaskProviderContext {
Expand All @@ -81,9 +80,7 @@ impl LazyFileScanTaskProvider {
if groups.is_empty() {
groups.push(Vec::new());
}
let mut provider = EagerFileScanTaskProvider::try_new(groups)?;
provider.statistics = Some(statistics);
Ok(provider)
EagerFileScanTaskProvider::try_new(groups)
}

/// Table retained for discovery, including its metadata and storage access.
Expand Down
19 changes: 7 additions & 12 deletions crates/datafusion/tests/file_scan_task_provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ async fn eager_scan_plans_only_the_matching_taxi_files() -> Result<()> {
}

#[tokio::test]
async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> {
async fn eager_count_skips_scan_when_no_files_match() -> Result<()> {
let harness = IcebergTestHarness::builder()
.with_iceberg_table_factory(IcebergTableProviderFactory::new())
.with_table(create_taxi_table().await?)
Expand All @@ -144,13 +144,8 @@ async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> {
.await?;

insta::assert_snapshot!(plan, @"
ProjectionExec: expr=[count(Int64(1))@0 as trips]
AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
CoalescePartitionsExec
AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
FilterExec: pickup_date@0 = 2025-01-01, projection=[]
RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
DataSourceExec: format=iceberg, projection=[pickup_date], provider=eager, planned_files=0
ProjectionExec: expr=[0 as trips]
PlaceholderRowExec
");
insta::assert_snapshot!(rows, @"
+-------+
Expand Down Expand Up @@ -196,7 +191,7 @@ async fn replacing_the_provider_preserves_the_fetch_limit() -> Result<()> {
}

#[tokio::test]
async fn eager_snapshot_statistics_apply_fetch_per_partition() -> Result<()> {
async fn eager_file_statistics_apply_fetch_per_partition() -> Result<()> {
let fixtures = FixtureCatalog::try_new().await?;
let schema = Schema::builder()
.with_fields(vec![
Expand Down Expand Up @@ -231,12 +226,12 @@ async fn eager_snapshot_statistics_apply_fetch_per_partition() -> Result<()> {
.build()
.await?;

// Eager conversion retains the six-row snapshot estimate for the assignment.
// Each of the two file groups contributes three rows to the assignment.
let plan = harness.physical_plan("SELECT id FROM items").await?;
insta::assert_snapshot!(harness.source_statistics(&plan)?, @"
all partitions (2): Rows=Inexact(6), Bytes=Inexact(48), [(Col[0]:)]
partition 0: Rows=Absent, Bytes=Absent, [(Col[0]:)]
partition 1: Rows=Absent, Bytes=Absent, [(Col[0]:)]
partition 0: Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]:)]
partition 1: Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]:)]
");

// Fetch caps each source partition at two rows, not the whole assignment.
Expand Down
37 changes: 36 additions & 1 deletion crates/datafusion/tests/statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
#[cfg(test)]
mod tests {
use datafusion::error::Result;
use iceberg_datafusion::IcebergTableProviderFactory;
use iceberg_datafusion::{IcebergDataFusionConfig, IcebergTableProviderFactory};
use iceberg_datafusion_test_tools::{IcebergTestHarness, create_taxi_table};

#[tokio::test]
Expand All @@ -43,4 +43,39 @@ mod tests {

Ok(())
}

#[tokio::test]
async fn eager_statistics_count_only_the_selected_files() -> Result<()> {
// Taxi has seven daily files of 25,000 rows; the predicate selects two.
let harness = IcebergTestHarness::builder()
.with_iceberg_table_factory(IcebergTableProviderFactory::new())
.with_table(create_taxi_table().await?)
.with_session_extension(|mut state| {
state
.config()
.get_or_insert_default()
.options_mut()
.extensions
.insert(IcebergDataFusionConfig::default());
state
})
.with_config_option("iceberg.execution.eager_file_planning", true)
.with_config_option("datafusion.execution.target_partitions", 2)
.build()
.await?;

let plan = harness
.physical_plan(
"SELECT vendor_id FROM taxi \
WHERE pickup_date IN (DATE '2024-01-10', DATE '2024-01-11')",
)
.await?;

insta::assert_snapshot!(harness.source_statistics(&plan)?, @"
all partitions (2): Rows=Inexact(50000), Bytes=Inexact(400000), [(Col[0]:),(Col[1]:)]
partition 0: Rows=Inexact(25000), Bytes=Inexact(200000), [(Col[0]:),(Col[1]:)]
partition 1: Rows=Inexact(25000), Bytes=Inexact(200000), [(Col[0]:),(Col[1]:)]
");
Ok(())
}
}