diff --git a/Cargo.lock b/Cargo.lock index b111a48..ffab806 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2657,6 +2657,9 @@ name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +dependencies = [ + "getrandom 0.4.3", +] [[package]] name = "find-msvc-tools" @@ -2691,6 +2694,18 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "fastrand", + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3243,6 +3258,7 @@ dependencies = [ "bytes", "dashmap", "datafusion", + "flume", "futures", "iceberg", "iceberg-datafusion-test-tools", @@ -5101,6 +5117,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + [[package]] name = "sqllogictest" version = "0.29.1" @@ -5608,7 +5633,7 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.5", + "rand 0.10.4", ] [[package]] diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml index e7fb281..2d85023 100644 --- a/crates/datafusion/Cargo.toml +++ b/crates/datafusion/Cargo.toml @@ -30,6 +30,7 @@ repository = { workspace = true } autotests = false [dependencies] +flume = "0.12" async-trait = { workspace = true } bytes = "1" dashmap = "6" diff --git a/crates/datafusion/src/read/file_scan_task_provider/eager.rs b/crates/datafusion/src/read/file_scan_task_provider/eager.rs index bbdd672..f80de8a 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/eager.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/eager.rs @@ -20,6 +20,7 @@ use datafusion::common::stats::Precision; use datafusion::common::{Result, Statistics, exec_err}; use datafusion::physical_plan::Partitioning; use futures::StreamExt; +use futures::stream::BoxStream; use iceberg::scan::FileScanTask; use std::collections::BTreeSet; use std::fmt::Formatter; @@ -70,7 +71,7 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider { self: Arc, partition: usize, _context: FileScanTaskProviderContext, - ) -> Result { + ) -> Result>> { let Some(count) = self.groups().get(partition).map(Vec::len) else { return exec_err!("Invalid Iceberg scan partition {partition}"); }; diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index 2a5b189..356275d 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -14,18 +14,24 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +use std::any::Any; use std::fmt::Formatter; +use std::ops::DerefMut; use std::sync::Arc; use datafusion::arrow::datatypes::{Schema, SchemaRef}; +use datafusion::common::runtime::SpawnedTask; use datafusion::common::stats::Precision; -use datafusion::common::{Result, Statistics, exec_err}; +use datafusion::common::{ + DataFusionError, Result, Statistics, exec_datafusion_err, exec_err, +}; use datafusion::execution::TaskContext; use datafusion::physical_plan::Partitioning; use datafusion::prelude::Expr; -use futures::TryStreamExt; +use futures::stream::BoxStream; +use futures::{Stream, StreamExt, TryStreamExt}; use iceberg::expr::Predicate; -use iceberg::scan::TableScan; +use iceberg::scan::{FileScanTask, FileScanTaskStream, TableScan}; use iceberg::spec::SnapshotRef; use iceberg::table::Table; @@ -48,6 +54,51 @@ pub struct LazyFileScanTaskProvider { snapshot_id: Option, projection: Vec, predicate: Option, + partitions: usize, +} + +#[derive(Debug, Default)] +enum StreamState { + #[default] + NotStarted, + Errored(Arc), + PartitionedFanout(PartitionedFanout), +} + +#[derive(Debug)] +struct PartitionedFanout { + task: Arc>, + per_partition_feed: Vec>>, +} + +impl PartitionedFanout { + fn new(partitions: usize, mut stream: FileScanTaskStream) -> Self { + let (tx, rx) = flume::unbounded(); + let task = SpawnedTask::spawn(async move { + while let Some(result) = stream.next().await { + // Every partition dropped its feed, e.g. under LIMIT or when the + // query was cancelled, so stop discovering files. + if tx.send(result.map_err(to_datafusion_error)).is_err() { + break; + } + } + }); + Self { + task: Arc::new(task), + per_partition_feed: vec![rx; partitions], + } + } + + fn pop_stream(&mut self) -> Result> + use<>> { + let Some(feed) = self.per_partition_feed.pop() else { + return exec_err!("Requested more FileScanTask streams than available"); + }; + let task = Arc::clone(&self.task); + Ok(feed.into_stream().inspect(move |_| { + // Keep the task alive as long as the stream is in scope. + let _ = &task; + })) + } } impl LazyFileScanTaskProvider { @@ -65,12 +116,8 @@ impl LazyFileScanTaskProvider { "Iceberg eager planning requires at least one target partition" ); } - let mut tasks = Arc::new(self).open( - 0, - FileScanTaskProviderContext { - task_context: &context, - }, - )?; + let mut tasks = self.scan.plan_files().await.map_err(to_datafusion_error)?; + let mut groups = Vec::new(); let mut partition = 0; while let Some(task) = tasks.try_next().await.map_err(to_datafusion_error)? { @@ -232,6 +279,7 @@ impl FileScanTaskProviderBuilder { projection, snapshot_id: self.snapshot_id, predicate: self.predicate, + partitions: self.context.session_config().target_partitions(), }; let provider: Arc = if IcebergDataFusionConfig::from_config_or_default( @@ -250,22 +298,53 @@ impl FileScanTaskProviderBuilder { impl FileScanTaskProvider for LazyFileScanTaskProvider { fn partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) + Partitioning::UnknownPartitioning(self.partitions) + } + + fn create_sibling_state(&self) -> Option> { + Some(Arc::new(tokio::sync::Mutex::::default())) } fn open( self: Arc, - partition: usize, - _ctx: FileScanTaskProviderContext, - ) -> Result { - if partition != 0 { - return exec_err!("Invalid Iceberg scan partition {partition}"); - } + _partition: usize, + ctx: FileScanTaskProviderContext, + ) -> Result>> { let scan = Arc::clone(&self.scan); - // Files are discovered when the stream is first polled, not here. - Ok(Box::pin( - futures::stream::once(async move { scan.plan_files().await }).try_flatten(), - )) + let partitions = self.partitions; + let state = ctx + .sibling_state + .and_then(|state| state.downcast::>().ok()) + .ok_or_else(|| { + exec_datafusion_err!("Missing sibling state for LazyFileScanTaskProvider") + })?; + + // Files are discovered when the first stream is first polled, not here. + let stream = async move { + let mut state = state.lock().await; + match state.deref_mut() { + StreamState::NotStarted => { + let file_scan_task_stream = match scan.plan_files().await { + Ok(stream) => stream, + Err(err) => { + let err = Arc::new(to_datafusion_error(err)); + *state = StreamState::Errored(Arc::clone(&err)); + return Err(DataFusionError::Shared(err)); + } + }; + let mut partitioned_fanout = + PartitionedFanout::new(partitions, file_scan_task_stream); + let file_scan_task_stream = partitioned_fanout.pop_stream()?; + *state = StreamState::PartitionedFanout(partitioned_fanout); + Ok(file_scan_task_stream) + } + StreamState::Errored(err) => { + Err(DataFusionError::Shared(Arc::clone(err))) + } + StreamState::PartitionedFanout(fanout) => fanout.pop_stream(), + } + }; + Ok(futures::stream::once(stream).try_flatten().boxed()) } /// Estimates the rows the scan returns before its predicate and fetch, diff --git a/crates/datafusion/src/read/file_scan_task_provider/provider.rs b/crates/datafusion/src/read/file_scan_task_provider/provider.rs index 1f8ba15..452ba43 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/provider.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/provider.rs @@ -22,6 +22,8 @@ use datafusion::arrow::datatypes::Schema; use datafusion::common::{Result, Statistics}; use datafusion::execution::TaskContext; use datafusion::physical_plan::Partitioning; +use futures::stream::BoxStream; +use iceberg::scan::FileScanTask; /// Supplies the file tasks an [`IcebergDataSource`](crate::read::IcebergDataSource) /// reads, split into output partitions. @@ -45,13 +47,21 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync { /// projected schema. fn partitioning(&self) -> Partitioning; + /// Creates state shared by the partitions of one execution, passed to + /// [`Self::open`] through [`FileScanTaskProviderContext::sibling_state`]. + /// The data source creates it once per execution, so re-executing a plan + /// after resetting its state starts afresh. Returns `None` by default. + fn create_sibling_state(&self) -> Option> { + None + } + /// Streams the file tasks of `partition`, which is below the /// [`Self::partitioning`] partition count. fn open( self: Arc, partition: usize, context: FileScanTaskProviderContext, - ) -> Result; + ) -> Result>>; /// Statistics before the source's fetch, without discovering or reading files. /// `None` requests the whole assignment; `Some(i)` requests partition `i`, @@ -85,4 +95,7 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync { /// table, belong to the provider rather than the data source. pub struct FileScanTaskProviderContext<'a> { pub task_context: &'a Arc, + /// The state from [`FileScanTaskProvider::create_sibling_state`] for this + /// execution, or `None` when the caller opens partitions on their own. + pub sibling_state: Option>, } diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index f74e8d1..18bed3e 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -15,17 +15,19 @@ // specific language governing permissions and limitations // under the License. +use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; use super::file_scan_task_provider::{FileScanTaskProvider, FileScanTaskProviderContext}; -use crate::to_datafusion_error; +use crate::{from_datafusion_error, to_datafusion_error}; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::stats::Precision; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{Statistics, exec_err}; -use datafusion::datasource::source::DataSource; +use datafusion::config::ConfigOptions; +use datafusion::datasource::source::{DataSource, OpenArgs}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::projection::ProjectionExprs; @@ -35,7 +37,7 @@ use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSe use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayFormatType, Partitioning}; use datafusion::prelude::SessionConfig; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use iceberg::arrow::{ArrowReader, ArrowReaderBuilder}; use iceberg::io::FileIO; @@ -231,6 +233,22 @@ impl DataSource for IcebergDataSource { partition: usize, context: Arc, ) -> Result { + self.open_with_args(OpenArgs::new(partition, context)) + } + + fn create_sibling_state( + &self, + _config: &ConfigOptions, + ) -> Option> { + self.file_scan_task_provider.create_sibling_state() + } + + fn open_with_args(&self, args: OpenArgs) -> Result { + let OpenArgs { + partition, + context, + sibling_state, + } = args; if partition >= self.output_partitioning().partition_count() { return exec_err!("Invalid Iceberg scan partition {partition}"); } @@ -238,12 +256,13 @@ impl DataSource for IcebergDataSource { partition, FileScanTaskProviderContext { task_context: &context, + sibling_state, }, )?; let stream = self .reader .clone() - .read(tasks) + .read(tasks.map_err(from_datafusion_error).boxed()) .map_err(to_datafusion_error)? .stream() .map_err(to_datafusion_error); @@ -344,7 +363,7 @@ mod tests { use datafusion::common::Statistics; use datafusion::common::stats::Precision; use datafusion::datasource::source::DataSource; - use datafusion::prelude::SessionContext; + use datafusion::prelude::{SessionConfig, SessionContext}; use iceberg::io::FileIO; use iceberg::spec::{ FormatVersion, MAIN_BRANCH, NestedField, Operation, PartitionSpec, PrimitiveType, @@ -420,7 +439,11 @@ mod tests { Field::new("id", DataType::Int64, false), Field::new("name", DataType::Utf8, true), ])); - let context = SessionContext::new(); + // One partition, so the fetch caps the whole scan rather than each of + // a machine-dependent number of partitions. + let context = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(1), + ); let (schema, provider) = FileScanTaskProviderBuilder::new(context.task_ctx(), table.clone(), schema) .with_projection(Some(&vec![0])) diff --git a/crates/datafusion/tests/catalog_provider.rs b/crates/datafusion/tests/catalog_provider.rs index c80934d..4e15db1 100644 --- a/crates/datafusion/tests/catalog_provider.rs +++ b/crates/datafusion/tests/catalog_provider.rs @@ -24,6 +24,8 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; use datafusion::execution::context::SessionContext; use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::{NamespaceIdent, TableIdent}; use iceberg_datafusion::IcebergCatalogProvider; @@ -77,8 +79,7 @@ async fn test_provider_plan_stream_schema() -> Result<(), Box> { .unwrap(); let task_ctx = Arc::new(df.task_ctx()); - let plan = df.create_physical_plan().await.unwrap(); - assert_eq!(plan.properties().partitioning.partition_count(), 1); + let plan = CoalescePartitionsExec::new(df.create_physical_plan().await.unwrap()); let stream = plan.execute(0, task_ctx).unwrap(); // Ensure both the plan and the stream conform to the same schema diff --git a/crates/datafusion/tests/file_scan_task_provider.rs b/crates/datafusion/tests/file_scan_task_provider.rs index c9c3447..a8a6527 100644 --- a/crates/datafusion/tests/file_scan_task_provider.rs +++ b/crates/datafusion/tests/file_scan_task_provider.rs @@ -20,8 +20,7 @@ use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch}; use datafusion::common::Result; -use datafusion::datasource::source::{DataSource, DataSourceExec}; -use datafusion::execution::TaskContext; +use datafusion::datasource::source::{DataSource, DataSourceExec, OpenArgs}; use datafusion::physical_plan::{ExecutionPlan, displayable}; use futures::TryStreamExt; use iceberg::TableIdent; @@ -49,7 +48,7 @@ async fn lazy_scan_filters_and_projects_taxi_rows() -> Result<()> { ) .await?; - insta::assert_snapshot!(plan, @" + insta::assert_snapshot!(plan, @r" SortPreservingMergeExec: [pickup_date@0 ASC NULLS LAST] ProjectionExec: expr=[pickup_date@0 as pickup_date, count(Int64(1))@1 as trips] SortExec: expr=[pickup_date@0 ASC NULLS LAST], preserve_partitioning=[true] @@ -57,8 +56,7 @@ async fn lazy_scan_filters_and_projects_taxi_rows() -> Result<()> { RepartitionExec: partitioning=Hash([pickup_date@0], 2), input_partitions=2 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 = 2024-01-10 OR pickup_date@0 = 2024-01-11 - RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) "); insta::assert_snapshot!(rows, @" +-------------+-------+ @@ -260,8 +258,13 @@ async fn the_scan_keeps_its_construction_time_reader_settings() -> Result<()> { let source = iceberg_source(&plan).unwrap(); // Open with default settings, not the settings used to construct the scan. + let sibling_state = source.create_sibling_state(harness.state().config_options()); let batches = source - .open(0, Arc::new(TaskContext::default()))? + .open_with_args(OpenArgs { + partition: 0, + context: Arc::new(Default::default()), + sibling_state, + })? .try_collect::>() .await?; diff --git a/crates/datafusion/tests/filter_pushdown.rs b/crates/datafusion/tests/filter_pushdown.rs index 47718c7..8a12409 100644 --- a/crates/datafusion/tests/filter_pushdown.rs +++ b/crates/datafusion/tests/filter_pushdown.rs @@ -49,8 +49,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: pickup_date@2 = 2024-01-10 AND (payment_type@1 = 1 OR payment_type@1 = 2) AND trip_distance@0 >= 2, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, payment_type, pickup_date], provider=lazy, predicate=((pickup_date = 2024-01-10) AND ((payment_type = 1) OR (payment_type = 2))) AND (trip_distance >= 2) + DataSourceExec: format=iceberg, projection=[trip_distance, payment_type, pickup_date], provider=lazy, predicate=((pickup_date = 2024-01-10) AND ((payment_type = 1) OR (payment_type = 2))) AND (trip_distance >= 2) "); insta::assert_snapshot!(batches, @r" +-------+ @@ -86,8 +85,7 @@ mod tests { RepartitionExec: partitioning=Hash([pickup_date@0], 16), input_partitions=16 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 = 2024-01-10 OR pickup_date@0 = 2024-01-11 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) "); insta::assert_snapshot!(batches, @r" +-------------+-------+ @@ -115,8 +113,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: passenger_count@0 IS NULL, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[passenger_count], provider=lazy, predicate=passenger_count IS NULL + DataSourceExec: format=iceberg, projection=[passenger_count], provider=lazy, predicate=passenger_count IS NULL "); insta::assert_snapshot!(batches, @r" +-------+ @@ -148,8 +145,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: pickup_date@1 = 2024-01-10 AND trip_distance@0 + 1 > 3, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------+ @@ -176,8 +172,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: trip_distance@0 + 1 > 20, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance], provider=lazy + DataSourceExec: format=iceberg, projection=[trip_distance], provider=lazy "); insta::assert_snapshot!(batches, @r" +-------+ diff --git a/crates/datafusion/tests/limit_pushdown.rs b/crates/datafusion/tests/limit_pushdown.rs index 9a721db..1572cc0 100644 --- a/crates/datafusion/tests/limit_pushdown.rs +++ b/crates/datafusion/tests/limit_pushdown.rs @@ -43,8 +43,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST], fetch=3 SortExec: TopK(fetch=3), expr=[pickup_at@1 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@4 = 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, passenger_count, trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, passenger_count, trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+-----------------+---------------+-------------+ @@ -70,8 +69,7 @@ mod tests { insta::assert_snapshot!(plan, @r" CoalescePartitionsExec: fetch=3 FilterExec: pickup_date@0 = 2024-01-10, fetch=3 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------+ @@ -91,22 +89,21 @@ mod tests { let (plan, batches) = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) - .query("SELECT pickup_date FROM taxi LIMIT 3") + .build() + .await? + .query_batches("SELECT pickup_date FROM taxi LIMIT 3") .await?; insta::assert_snapshot!(plan, @r" - CooperativeExec + CoalescePartitionsExec: fetch=3 DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, fetch=3 "); - insta::assert_snapshot!(batches, @r" - +-------------+ - | pickup_date | - +-------------+ - | 2024-01-08 | - | 2024-01-08 | - | 2024-01-08 | - +-------------+ - "); + // No ORDER BY, so with several partitions any day's file can supply + // the rows; only their count is deterministic. + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 3 + ); Ok(()) } @@ -116,20 +113,24 @@ mod tests { let (_, explain) = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) + // With several partitions, how many open a file before the limit + // cancels the scan depends on scheduling. One partition stops after + // the first file, so the source's row count is deterministic. + .with_config_option("datafusion.execution.target_partitions", 1) // Only row counts, so the output doesn't vary between runs. .with_config_option("datafusion.explain.analyze_categories", "rows") .query("EXPLAIN ANALYZE SELECT pickup_date FROM taxi LIMIT 3") .await?; insta::assert_snapshot!(explain, @r" - +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ - | plan_type | plan | - +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ - | Plan with Metrics | CooperativeExec, metrics=[] | - | | DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, fetch=3, metrics=[output_rows=3, output_batches=1, batches_split=0] | - | | | - +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ - "); + +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ + | plan_type | plan | + +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ + | Plan with Metrics | CooperativeExec, metrics=[] | + | | DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, fetch=3, metrics=[output_rows=3, output_batches=1, batches_split=0] | + | | | + +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ + "); Ok(()) } @@ -144,6 +145,7 @@ mod tests { )), ) .with_table(create_taxi_table().await?) + .with_config_option("datafusion.execution.target_partitions", 2) .build() .await?; harness.query_batches("SELECT * FROM taxi").await?; @@ -151,8 +153,8 @@ mod tests { harness.query_batches("SELECT * FROM taxi LIMIT 3").await?; let limited_scan_reads = storage.take_reads(); - // The exact counts depend on how many data files iceberg opens at - // once, which follows the runner's CPU count, so compare the two scans. + // Two partitions can open fewer than the seven files before LIMIT + // cancels the scan. Compare totals, not scheduler-dependent exact counts. assert!( limited_scan_reads < full_scan_reads, "a scan with LIMIT 3 read {limited_scan_reads} Parquet ranges, \ @@ -201,8 +203,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@2 ASC NULLS LAST], fetch=5 SortExec: TopK(fetch=5), expr=[pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@2 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@3 = 2024-01-10, projection=[vendor_id@0, pickup_at@1, pickup_location_id@2] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+ diff --git a/crates/datafusion/tests/plan_nodes.rs b/crates/datafusion/tests/plan_nodes.rs index 91b028a..c2c75a7 100644 --- a/crates/datafusion/tests/plan_nodes.rs +++ b/crates/datafusion/tests/plan_nodes.rs @@ -106,8 +106,10 @@ async fn run_source( source: &IcebergDataSource, ctx: &SessionContext, ) -> Result> { - let exec = DataSourceExec::from_data_source(source.clone()); - run(exec.as_ref(), ctx).await + // The lazy provider reads in one partition per target partition. + let exec = + CoalescePartitionsExec::new(DataSourceExec::from_data_source(source.clone())); + run(&exec, ctx).await } /// Returns the first node of type `T` in `plan`, depth first. diff --git a/crates/datafusion/tests/projection_pushdown.rs b/crates/datafusion/tests/projection_pushdown.rs index dc4f996..0e249e9 100644 --- a/crates/datafusion/tests/projection_pushdown.rs +++ b/crates/datafusion/tests/projection_pushdown.rs @@ -51,8 +51,7 @@ mod tests { RepartitionExec: partitioning=Hash([pickup_date@0], 16), input_partitions=16 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 >= 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date >= 2024-01-10 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date >= 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------+-------+ @@ -88,8 +87,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@2 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST], fetch=3 SortExec: TopK(fetch=3), expr=[pickup_at@2 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@3 = 2024-01-10, projection=[vendor_id@0, pickup_location_id@2, pickup_at@1] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+--------------------+ @@ -122,8 +120,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[max(taxi.trip_distance * taxi.fare_amount)] FilterExec: pickup_date@2 = 2024-01-10, projection=[trip_distance@0, fare_amount@1] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, fare_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[trip_distance, fare_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------------+ @@ -154,8 +151,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@5 ASC NULLS LAST], fetch=1 SortExec: TopK(fetch=1), expr=[pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@5 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@12 = 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, dropoff_at, passenger_count, trip_distance, pickup_location_id, dropoff_location_id, payment_type, fare_amount, tip_amount, tolls_amount, total_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, dropoff_at, passenger_count, trip_distance, pickup_location_id, dropoff_location_id, payment_type, fare_amount, tip_amount, tolls_amount, total_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+---------------------+-----------------+---------------+--------------------+---------------------+--------------+-------------+------------+--------------+--------------+-------------+ diff --git a/crates/datafusion/tests/schema_evolution.rs b/crates/datafusion/tests/schema_evolution.rs index bb865a5..3326641 100644 --- a/crates/datafusion/tests/schema_evolution.rs +++ b/crates/datafusion/tests/schema_evolution.rs @@ -24,6 +24,8 @@ use datafusion::arrow::compute::{concat_batches, sort_to_indices, take_record_ba use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::datasource::TableProvider; use datafusion::execution::context::SessionContext; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::common::collect; use iceberg::TableIdent; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; @@ -100,7 +102,7 @@ async fn test_scan_after_schema_evolution_reads_provider_columns() // The rows come from two data files, which may be read in either order. // They are joined under their own schema, not the plan's, so that a column // the plan does not report would show. - assert_eq!(plan.properties().partitioning.partition_count(), 1); + let plan = CoalescePartitionsExec::new(plan); let batches = collect(plan.execute(0, ctx.task_ctx())?).await?; let batch = concat_batches(&batches[0].schema(), &batches)?; let order = sort_to_indices(batch.column_by_name("foo1").unwrap(), None, None)?; diff --git a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt b/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt index bac4267..9636326 100644 --- a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt +++ b/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt @@ -51,7 +51,7 @@ logical_plan 01)Limit: skip=0, fetch=3 02)--TableScan: default.default.query_test_table projection=[id, name, score, category], fetch=3 physical_plan -01)CooperativeExec +01)CoalescePartitionsExec: fetch=3 02)--DataSourceExec: format=iceberg, projection=[id, name, score, category], provider=lazy, fetch=3 # Test SELECT * with ORDER BY and LIMIT diff --git a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt index 6a22dd9..205be33 100644 --- a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt @@ -38,7 +38,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = true # Query with is_active = true @@ -58,7 +58,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = false # Query with is_active = false @@ -77,7 +77,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = false # Query with is_active != true (includes false and NULL) diff --git a/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt index d25e445..611172f 100644 --- a/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt @@ -40,7 +40,7 @@ logical_plan 03)----TableScan: default.default.test_cast_table projection=[id, d], partial_filters=[CAST(default.default.test_cast_table.d AS Timestamp(ns)) < TimestampNanosecond(1672920000000000000, None)] physical_plan 01)FilterExec: CAST(d@1 AS Timestamp(ns)) < 1672920000000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, d], provider=lazy query I rowsort @@ -73,7 +73,7 @@ logical_plan 03)----TableScan: default.default.test_cast_table projection=[id, n], partial_filters=[CAST(default.default.test_cast_table.n AS Utf8) < Utf8("5")] physical_plan 01)FilterExec: CAST(n@1 AS Utf8) < 5, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, n], provider=lazy query I rowsort @@ -108,7 +108,7 @@ logical_plan 03)----TableScan: default.default.test_float_cast_table projection=[id, x], partial_filters=[CAST(default.default.test_float_cast_table.x AS Float64) > Float64(0.1)] physical_plan 01)FilterExec: CAST(x@1 AS Float64) > 0.1, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, x], provider=lazy query I rowsort @@ -137,7 +137,7 @@ logical_plan 03)----TableScan: default.default.test_isnan_cast_table projection=[id, s], partial_filters=[NOT isnan(CAST(default.default.test_isnan_cast_table.s AS Float64))] physical_plan 01)FilterExec: NOT isnan(CAST(s@1 AS Float64)), projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, s], provider=lazy query I rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt index dc4c36a..d3c93f6 100644 --- a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt @@ -36,7 +36,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name LIKE Utf8("Al%")] physical_plan 01)FilterExec: name@1 LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name STARTS WITH "Al" # Test LIKE filtering with case-sensitive match @@ -55,7 +55,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name NOT LIKE Utf8("Al%")] physical_plan 01)FilterExec: name@1 NOT LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name NOT STARTS WITH "Al" # Test NOT LIKE filtering @@ -129,7 +129,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name LIKE Utf8("Al\%%")] physical_plan 01)FilterExec: name@1 LIKE Al\%% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name STARTS WITH "Al%" query IT rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt index a23ad57..166368a 100644 --- a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt @@ -49,7 +49,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts = TimestampNanosecond(1672921800000000000, None)] physical_plan 01)FilterExec: ts@1 = 1672921800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts = 2023-01-05 12:30:00 # Verify timestamp equality filtering works @@ -67,7 +67,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts > TimestampNanosecond(1673308800000000000, None)] physical_plan 01)FilterExec: ts@1 > 1673308800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts > 2023-01-10 00:00:00 # Verify timestamp greater than filtering @@ -96,7 +96,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts >= TimestampNanosecond(1672876800000000000, None), default.default.test_timestamp_table.ts <= TimestampNanosecond(1673827199000000000, None)] physical_plan 01)FilterExec: ts@1 >= 1672876800000000000 AND ts@1 <= 1673827199000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59) # Test timestamp range predicate filtering @@ -161,7 +161,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_micros projection=[id, ts], partial_filters=[default.default.test_timestamp_micros.ts > TimestampMicrosecond(1672531200000000, None)] physical_plan 01)FilterExec: ts@1 > 1672531200000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts > 2023-01-01 00:00:00 query I? @@ -206,7 +206,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_us], partial_filters=[default.default.test_timestamptz.ts_us > TimestampMicrosecond(1672876800000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_us@1 > 1672876800000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_us], provider=lazy, predicate=ts_us > 2023-01-05 00:00:00 UTC query I @@ -223,7 +223,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_ns], partial_filters=[default.default.test_timestamptz.ts_ns > TimestampNanosecond(1672876800000000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_ns@1 > 1672876800000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_ns], provider=lazy, predicate=ts_ns > 2023-01-05 00:00:00 UTC query I @@ -240,7 +240,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_us], partial_filters=[default.default.test_timestamptz.ts_us > TimestampMicrosecond(1672876800000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_us@1 > 1672876800000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_us], provider=lazy, predicate=ts_us > 2023-01-05 00:00:00 UTC query I @@ -257,7 +257,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_naive], partial_filters=[default.default.test_timestamptz.ts_naive > TimestampNanosecond(1672876800000000000, None)] physical_plan 01)FilterExec: ts_naive@1 > 1672876800000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_naive], provider=lazy, predicate=ts_naive > 2023-01-05 00:00:00 query I