From d3fec8ec2bc0b936505c29d02a052d52d9ed813b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 20 Nov 2026 11:41:18 +0100 Subject: [PATCH 1/3] feat: read lazy Iceberg scans across target partitions A lazy scan exposed a single output partition, so DataFusion had one scan partition to schedule no matter how many files the scan matched. It now exposes the session's target partitions. One background task drains TableScan::plan_files() into a shared channel, and every partition pulls the next file from it, so the first file can be read while the rest are still being planned. Discovery stops once every partition has dropped its feed. FileScanTaskProvider::open now yields DataFusion results, converting to iceberg errors only where the Arrow reader needs them. --- Cargo.lock | 27 +++- crates/datafusion/Cargo.toml | 1 + .../src/read/file_scan_task_provider/eager.rs | 3 +- .../src/read/file_scan_task_provider/lazy.rs | 125 +++++++++++++++--- .../read/file_scan_task_provider/provider.rs | 4 +- crates/datafusion/src/read/scan.rs | 6 +- 6 files changed, 140 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b111a48..ffab806 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2657,6 +2657,9 @@ name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +dependencies = [ + "getrandom 0.4.3", +] [[package]] name = "find-msvc-tools" @@ -2691,6 +2694,18 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "fastrand", + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3243,6 +3258,7 @@ dependencies = [ "bytes", "dashmap", "datafusion", + "flume", "futures", "iceberg", "iceberg-datafusion-test-tools", @@ -5101,6 +5117,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + [[package]] name = "sqllogictest" version = "0.29.1" @@ -5608,7 +5633,7 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.5", + "rand 0.10.4", ] [[package]] diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml index e7fb281..2d85023 100644 --- a/crates/datafusion/Cargo.toml +++ b/crates/datafusion/Cargo.toml @@ -30,6 +30,7 @@ repository = { workspace = true } autotests = false [dependencies] +flume = "0.12" async-trait = { workspace = true } bytes = "1" dashmap = "6" diff --git a/crates/datafusion/src/read/file_scan_task_provider/eager.rs b/crates/datafusion/src/read/file_scan_task_provider/eager.rs index bbdd672..f80de8a 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/eager.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/eager.rs @@ -20,6 +20,7 @@ use datafusion::common::stats::Precision; use datafusion::common::{Result, Statistics, exec_err}; use datafusion::physical_plan::Partitioning; use futures::StreamExt; +use futures::stream::BoxStream; use iceberg::scan::FileScanTask; use std::collections::BTreeSet; use std::fmt::Formatter; @@ -70,7 +71,7 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider { self: Arc, partition: usize, _context: FileScanTaskProviderContext, - ) -> Result { + ) -> Result>> { let Some(count) = self.groups().get(partition).map(Vec::len) else { return exec_err!("Invalid Iceberg scan partition {partition}"); }; diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index 2a5b189..87a32bf 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -15,17 +15,20 @@ // specific language governing permissions and limitations // under the License. use std::fmt::Formatter; +use std::ops::DerefMut; use std::sync::Arc; use datafusion::arrow::datatypes::{Schema, SchemaRef}; +use datafusion::common::runtime::SpawnedTask; use datafusion::common::stats::Precision; -use datafusion::common::{Result, Statistics, exec_err}; +use datafusion::common::{DataFusionError, Result, Statistics, exec_err}; use datafusion::execution::TaskContext; use datafusion::physical_plan::Partitioning; use datafusion::prelude::Expr; -use futures::TryStreamExt; +use futures::stream::BoxStream; +use futures::{Stream, StreamExt, TryStreamExt}; use iceberg::expr::Predicate; -use iceberg::scan::TableScan; +use iceberg::scan::{FileScanTask, FileScanTaskStream, TableScan}; use iceberg::spec::SnapshotRef; use iceberg::table::Table; @@ -40,7 +43,7 @@ const TOTAL_RECORDS: &str = "total-records"; /// Discovers files while executing, in a single partition: the files of a /// snapshot that may match a predicate. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct LazyFileScanTaskProvider { table: Table, scan: Arc, @@ -48,6 +51,68 @@ pub struct LazyFileScanTaskProvider { snapshot_id: Option, projection: Vec, predicate: Option, + partitions: usize, + state: Arc>, +} + +impl Clone for LazyFileScanTaskProvider { + fn clone(&self) -> Self { + Self { + table: self.table.clone(), + scan: Arc::clone(&self.scan), + schema: Arc::clone(&self.schema), + snapshot_id: self.snapshot_id, + projection: self.projection.clone(), + predicate: self.predicate.clone(), + partitions: self.partitions, + // A clone starts its own discovery rather than sharing this one's. + state: Default::default(), + } + } +} + +#[derive(Debug, Default)] +enum StreamState { + #[default] + NotStarted, + Errored(Arc), + PartitionedFanout(PartitionedFanout), +} + +#[derive(Debug)] +struct PartitionedFanout { + task: Arc>, + per_partition_feed: Vec>>, +} + +impl PartitionedFanout { + fn new(partitions: usize, mut stream: FileScanTaskStream) -> Self { + let (tx, rx) = flume::unbounded(); + let task = SpawnedTask::spawn(async move { + while let Some(result) = stream.next().await { + // Every partition dropped its feed, e.g. under LIMIT or when the + // query was cancelled, so stop discovering files. + if tx.send(result.map_err(to_datafusion_error)).is_err() { + break; + } + } + }); + Self { + task: Arc::new(task), + per_partition_feed: vec![rx; partitions], + } + } + + fn pop_stream(&mut self) -> Result> + use<>> { + let Some(feed) = self.per_partition_feed.pop() else { + return exec_err!("Requested more FileScanTask streams than available"); + }; + let task = Arc::clone(&self.task); + Ok(feed.into_stream().inspect(move |_| { + // Keep the task alive as long as the stream is in scope. + let _ = &task; + })) + } } impl LazyFileScanTaskProvider { @@ -65,12 +130,8 @@ impl LazyFileScanTaskProvider { "Iceberg eager planning requires at least one target partition" ); } - let mut tasks = Arc::new(self).open( - 0, - FileScanTaskProviderContext { - task_context: &context, - }, - )?; + let mut tasks = self.scan.plan_files().await.map_err(to_datafusion_error)?; + let mut groups = Vec::new(); let mut partition = 0; while let Some(task) = tasks.try_next().await.map_err(to_datafusion_error)? { @@ -232,6 +293,8 @@ impl FileScanTaskProviderBuilder { projection, snapshot_id: self.snapshot_id, predicate: self.predicate, + partitions: self.context.session_config().target_partitions(), + state: Default::default(), }; let provider: Arc = if IcebergDataFusionConfig::from_config_or_default( @@ -250,22 +313,44 @@ impl FileScanTaskProviderBuilder { impl FileScanTaskProvider for LazyFileScanTaskProvider { fn partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) + Partitioning::UnknownPartitioning(self.partitions) } fn open( self: Arc, - partition: usize, + _partition: usize, _ctx: FileScanTaskProviderContext, - ) -> Result { - if partition != 0 { - return exec_err!("Invalid Iceberg scan partition {partition}"); - } + ) -> Result>> { let scan = Arc::clone(&self.scan); - // Files are discovered when the stream is first polled, not here. - Ok(Box::pin( - futures::stream::once(async move { scan.plan_files().await }).try_flatten(), - )) + let partitions = self.partitions; + let state = Arc::clone(&self.state); + + // Files are discovered when the first stream is first polled, not here. + let stream = async move { + let mut state = state.lock().await; + match state.deref_mut() { + StreamState::NotStarted => { + let file_scan_task_stream = match scan.plan_files().await { + Ok(stream) => stream, + Err(err) => { + let err = Arc::new(to_datafusion_error(err)); + *state = StreamState::Errored(Arc::clone(&err)); + return Err(DataFusionError::Shared(err)); + } + }; + let mut partitioned_fanout = + PartitionedFanout::new(partitions, file_scan_task_stream); + let file_scan_task_stream = partitioned_fanout.pop_stream()?; + *state = StreamState::PartitionedFanout(partitioned_fanout); + Ok(file_scan_task_stream) + } + StreamState::Errored(err) => { + Err(DataFusionError::Shared(Arc::clone(err))) + } + StreamState::PartitionedFanout(fanout) => fanout.pop_stream(), + } + }; + Ok(futures::stream::once(stream).try_flatten().boxed()) } /// Estimates the rows the scan returns before its predicate and fetch, diff --git a/crates/datafusion/src/read/file_scan_task_provider/provider.rs b/crates/datafusion/src/read/file_scan_task_provider/provider.rs index 1f8ba15..3e13e91 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/provider.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/provider.rs @@ -22,6 +22,8 @@ use datafusion::arrow::datatypes::Schema; use datafusion::common::{Result, Statistics}; use datafusion::execution::TaskContext; use datafusion::physical_plan::Partitioning; +use futures::stream::BoxStream; +use iceberg::scan::FileScanTask; /// Supplies the file tasks an [`IcebergDataSource`](crate::read::IcebergDataSource) /// reads, split into output partitions. @@ -51,7 +53,7 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync { self: Arc, partition: usize, context: FileScanTaskProviderContext, - ) -> Result; + ) -> Result>>; /// Statistics before the source's fetch, without discovering or reading files. /// `None` requests the whole assignment; `Some(i)` requests partition `i`, diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index f74e8d1..2355c92 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -19,7 +19,7 @@ use std::fmt::{Debug, Formatter}; use std::sync::Arc; use super::file_scan_task_provider::{FileScanTaskProvider, FileScanTaskProviderContext}; -use crate::to_datafusion_error; +use crate::{from_datafusion_error, to_datafusion_error}; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::stats::Precision; @@ -35,7 +35,7 @@ use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSe use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayFormatType, Partitioning}; use datafusion::prelude::SessionConfig; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use iceberg::arrow::{ArrowReader, ArrowReaderBuilder}; use iceberg::io::FileIO; @@ -243,7 +243,7 @@ impl DataSource for IcebergDataSource { let stream = self .reader .clone() - .read(tasks) + .read(tasks.map_err(from_datafusion_error).boxed()) .map_err(to_datafusion_error)? .stream() .map_err(to_datafusion_error); From aa611e66ef3294daef6df7d2786bcb4a30ac7773 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 20 Nov 2026 11:41:46 +0100 Subject: [PATCH 2/3] feat: share lazy scan discovery through DataFusion sibling state The discovery state lived on the provider, so it outlived a single execution: running the same physical plan again found the fanout already drained. DataSourceExec already creates per-execution state shared by a source's partitions, and clears it on reset_state. FileScanTaskProvider gains a create_sibling_state hook that IcebergDataSource forwards to DataFusion, and the state reaches each partition's open through FileScanTaskProviderContext. --- .../src/read/file_scan_task_provider/lazy.rs | 38 ++++++++----------- .../read/file_scan_task_provider/provider.rs | 11 ++++++ crates/datafusion/src/read/scan.rs | 21 +++++++++- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index 87a32bf..356275d 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +use std::any::Any; use std::fmt::Formatter; use std::ops::DerefMut; use std::sync::Arc; @@ -21,7 +22,9 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{Schema, SchemaRef}; use datafusion::common::runtime::SpawnedTask; use datafusion::common::stats::Precision; -use datafusion::common::{DataFusionError, Result, Statistics, exec_err}; +use datafusion::common::{ + DataFusionError, Result, Statistics, exec_datafusion_err, exec_err, +}; use datafusion::execution::TaskContext; use datafusion::physical_plan::Partitioning; use datafusion::prelude::Expr; @@ -43,7 +46,7 @@ const TOTAL_RECORDS: &str = "total-records"; /// Discovers files while executing, in a single partition: the files of a /// snapshot that may match a predicate. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct LazyFileScanTaskProvider { table: Table, scan: Arc, @@ -52,23 +55,6 @@ pub struct LazyFileScanTaskProvider { projection: Vec, predicate: Option, partitions: usize, - state: Arc>, -} - -impl Clone for LazyFileScanTaskProvider { - fn clone(&self) -> Self { - Self { - table: self.table.clone(), - scan: Arc::clone(&self.scan), - schema: Arc::clone(&self.schema), - snapshot_id: self.snapshot_id, - projection: self.projection.clone(), - predicate: self.predicate.clone(), - partitions: self.partitions, - // A clone starts its own discovery rather than sharing this one's. - state: Default::default(), - } - } } #[derive(Debug, Default)] @@ -294,7 +280,6 @@ impl FileScanTaskProviderBuilder { snapshot_id: self.snapshot_id, predicate: self.predicate, partitions: self.context.session_config().target_partitions(), - state: Default::default(), }; let provider: Arc = if IcebergDataFusionConfig::from_config_or_default( @@ -316,14 +301,23 @@ impl FileScanTaskProvider for LazyFileScanTaskProvider { Partitioning::UnknownPartitioning(self.partitions) } + fn create_sibling_state(&self) -> Option> { + Some(Arc::new(tokio::sync::Mutex::::default())) + } + fn open( self: Arc, _partition: usize, - _ctx: FileScanTaskProviderContext, + ctx: FileScanTaskProviderContext, ) -> Result>> { let scan = Arc::clone(&self.scan); let partitions = self.partitions; - let state = Arc::clone(&self.state); + let state = ctx + .sibling_state + .and_then(|state| state.downcast::>().ok()) + .ok_or_else(|| { + exec_datafusion_err!("Missing sibling state for LazyFileScanTaskProvider") + })?; // Files are discovered when the first stream is first polled, not here. let stream = async move { diff --git a/crates/datafusion/src/read/file_scan_task_provider/provider.rs b/crates/datafusion/src/read/file_scan_task_provider/provider.rs index 3e13e91..452ba43 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/provider.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/provider.rs @@ -47,6 +47,14 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync { /// projected schema. fn partitioning(&self) -> Partitioning; + /// Creates state shared by the partitions of one execution, passed to + /// [`Self::open`] through [`FileScanTaskProviderContext::sibling_state`]. + /// The data source creates it once per execution, so re-executing a plan + /// after resetting its state starts afresh. Returns `None` by default. + fn create_sibling_state(&self) -> Option> { + None + } + /// Streams the file tasks of `partition`, which is below the /// [`Self::partitioning`] partition count. fn open( @@ -87,4 +95,7 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync { /// table, belong to the provider rather than the data source. pub struct FileScanTaskProviderContext<'a> { pub task_context: &'a Arc, + /// The state from [`FileScanTaskProvider::create_sibling_state`] for this + /// execution, or `None` when the caller opens partitions on their own. + pub sibling_state: Option>, } diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index 2355c92..717bc3d 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -25,7 +26,8 @@ use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::stats::Precision; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{Statistics, exec_err}; -use datafusion::datasource::source::DataSource; +use datafusion::config::ConfigOptions; +use datafusion::datasource::source::{DataSource, OpenArgs}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::projection::ProjectionExprs; @@ -231,6 +233,22 @@ impl DataSource for IcebergDataSource { partition: usize, context: Arc, ) -> Result { + self.open_with_args(OpenArgs::new(partition, context)) + } + + fn create_sibling_state( + &self, + _config: &ConfigOptions, + ) -> Option> { + self.file_scan_task_provider.create_sibling_state() + } + + fn open_with_args(&self, args: OpenArgs) -> Result { + let OpenArgs { + partition, + context, + sibling_state, + } = args; if partition >= self.output_partitioning().partition_count() { return exec_err!("Invalid Iceberg scan partition {partition}"); } @@ -238,6 +256,7 @@ impl DataSource for IcebergDataSource { partition, FileScanTaskProviderContext { task_context: &context, + sibling_state, }, )?; let stream = self From aa44991f1b2ccf801e02465f7e38db2ab67d64b2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 20 Nov 2026 11:42:23 +0100 Subject: [PATCH 3/3] test: update tests for partitioned lazy scans The lazy scan now produces one partition per target partition, so plans lose the round-robin repartition above the scan, and a plain LIMIT coalesces the scan's partitions instead of passing its single partition through. Tests that executed partition 0 of a single-partition scan now coalesce it first. Tests whose result depends on the partition count pin it: the fetch cap on whole-scan statistics and the EXPLAIN ANALYZE row counts under LIMIT use one partition, and the LIMIT read-count test uses two, so fewer than the seven taxi files can be open when the limit cancels the scan. A LIMIT without ORDER BY now only asserts how many rows come back, since any partition can supply them. --- crates/datafusion/src/read/scan.rs | 8 ++- crates/datafusion/tests/catalog_provider.rs | 5 +- .../tests/file_scan_task_provider.rs | 15 +++-- crates/datafusion/tests/filter_pushdown.rs | 15 ++--- crates/datafusion/tests/limit_pushdown.rs | 55 ++++++++++--------- crates/datafusion/tests/plan_nodes.rs | 6 +- .../datafusion/tests/projection_pushdown.rs | 12 ++-- crates/datafusion/tests/schema_evolution.rs | 4 +- .../testdata/slts/df_test/basic_queries.slt | 2 +- .../df_test/boolean_predicate_pushdown.slt | 6 +- .../slts/df_test/cast_predicate_pushdown.slt | 8 +-- .../slts/df_test/like_predicate_pushdown.slt | 6 +- .../df_test/timestamp_predicate_pushdown.slt | 16 +++--- 13 files changed, 81 insertions(+), 77 deletions(-) diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index 717bc3d..18bed3e 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -363,7 +363,7 @@ mod tests { use datafusion::common::Statistics; use datafusion::common::stats::Precision; use datafusion::datasource::source::DataSource; - use datafusion::prelude::SessionContext; + use datafusion::prelude::{SessionConfig, SessionContext}; use iceberg::io::FileIO; use iceberg::spec::{ FormatVersion, MAIN_BRANCH, NestedField, Operation, PartitionSpec, PrimitiveType, @@ -439,7 +439,11 @@ mod tests { Field::new("id", DataType::Int64, false), Field::new("name", DataType::Utf8, true), ])); - let context = SessionContext::new(); + // One partition, so the fetch caps the whole scan rather than each of + // a machine-dependent number of partitions. + let context = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(1), + ); let (schema, provider) = FileScanTaskProviderBuilder::new(context.task_ctx(), table.clone(), schema) .with_projection(Some(&vec![0])) diff --git a/crates/datafusion/tests/catalog_provider.rs b/crates/datafusion/tests/catalog_provider.rs index c80934d..4e15db1 100644 --- a/crates/datafusion/tests/catalog_provider.rs +++ b/crates/datafusion/tests/catalog_provider.rs @@ -24,6 +24,8 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; use datafusion::execution::context::SessionContext; use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::{NamespaceIdent, TableIdent}; use iceberg_datafusion::IcebergCatalogProvider; @@ -77,8 +79,7 @@ async fn test_provider_plan_stream_schema() -> Result<(), Box> { .unwrap(); let task_ctx = Arc::new(df.task_ctx()); - let plan = df.create_physical_plan().await.unwrap(); - assert_eq!(plan.properties().partitioning.partition_count(), 1); + let plan = CoalescePartitionsExec::new(df.create_physical_plan().await.unwrap()); let stream = plan.execute(0, task_ctx).unwrap(); // Ensure both the plan and the stream conform to the same schema diff --git a/crates/datafusion/tests/file_scan_task_provider.rs b/crates/datafusion/tests/file_scan_task_provider.rs index c9c3447..a8a6527 100644 --- a/crates/datafusion/tests/file_scan_task_provider.rs +++ b/crates/datafusion/tests/file_scan_task_provider.rs @@ -20,8 +20,7 @@ use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch}; use datafusion::common::Result; -use datafusion::datasource::source::{DataSource, DataSourceExec}; -use datafusion::execution::TaskContext; +use datafusion::datasource::source::{DataSource, DataSourceExec, OpenArgs}; use datafusion::physical_plan::{ExecutionPlan, displayable}; use futures::TryStreamExt; use iceberg::TableIdent; @@ -49,7 +48,7 @@ async fn lazy_scan_filters_and_projects_taxi_rows() -> Result<()> { ) .await?; - insta::assert_snapshot!(plan, @" + insta::assert_snapshot!(plan, @r" SortPreservingMergeExec: [pickup_date@0 ASC NULLS LAST] ProjectionExec: expr=[pickup_date@0 as pickup_date, count(Int64(1))@1 as trips] SortExec: expr=[pickup_date@0 ASC NULLS LAST], preserve_partitioning=[true] @@ -57,8 +56,7 @@ async fn lazy_scan_filters_and_projects_taxi_rows() -> Result<()> { RepartitionExec: partitioning=Hash([pickup_date@0], 2), input_partitions=2 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 = 2024-01-10 OR pickup_date@0 = 2024-01-11 - RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) "); insta::assert_snapshot!(rows, @" +-------------+-------+ @@ -260,8 +258,13 @@ async fn the_scan_keeps_its_construction_time_reader_settings() -> Result<()> { let source = iceberg_source(&plan).unwrap(); // Open with default settings, not the settings used to construct the scan. + let sibling_state = source.create_sibling_state(harness.state().config_options()); let batches = source - .open(0, Arc::new(TaskContext::default()))? + .open_with_args(OpenArgs { + partition: 0, + context: Arc::new(Default::default()), + sibling_state, + })? .try_collect::>() .await?; diff --git a/crates/datafusion/tests/filter_pushdown.rs b/crates/datafusion/tests/filter_pushdown.rs index 47718c7..8a12409 100644 --- a/crates/datafusion/tests/filter_pushdown.rs +++ b/crates/datafusion/tests/filter_pushdown.rs @@ -49,8 +49,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: pickup_date@2 = 2024-01-10 AND (payment_type@1 = 1 OR payment_type@1 = 2) AND trip_distance@0 >= 2, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, payment_type, pickup_date], provider=lazy, predicate=((pickup_date = 2024-01-10) AND ((payment_type = 1) OR (payment_type = 2))) AND (trip_distance >= 2) + DataSourceExec: format=iceberg, projection=[trip_distance, payment_type, pickup_date], provider=lazy, predicate=((pickup_date = 2024-01-10) AND ((payment_type = 1) OR (payment_type = 2))) AND (trip_distance >= 2) "); insta::assert_snapshot!(batches, @r" +-------+ @@ -86,8 +85,7 @@ mod tests { RepartitionExec: partitioning=Hash([pickup_date@0], 16), input_partitions=16 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 = 2024-01-10 OR pickup_date@0 = 2024-01-11 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) "); insta::assert_snapshot!(batches, @r" +-------------+-------+ @@ -115,8 +113,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: passenger_count@0 IS NULL, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[passenger_count], provider=lazy, predicate=passenger_count IS NULL + DataSourceExec: format=iceberg, projection=[passenger_count], provider=lazy, predicate=passenger_count IS NULL "); insta::assert_snapshot!(batches, @r" +-------+ @@ -148,8 +145,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: pickup_date@1 = 2024-01-10 AND trip_distance@0 + 1 > 3, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------+ @@ -176,8 +172,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: trip_distance@0 + 1 > 20, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance], provider=lazy + DataSourceExec: format=iceberg, projection=[trip_distance], provider=lazy "); insta::assert_snapshot!(batches, @r" +-------+ diff --git a/crates/datafusion/tests/limit_pushdown.rs b/crates/datafusion/tests/limit_pushdown.rs index 9a721db..1572cc0 100644 --- a/crates/datafusion/tests/limit_pushdown.rs +++ b/crates/datafusion/tests/limit_pushdown.rs @@ -43,8 +43,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST], fetch=3 SortExec: TopK(fetch=3), expr=[pickup_at@1 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@4 = 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, passenger_count, trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, passenger_count, trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+-----------------+---------------+-------------+ @@ -70,8 +69,7 @@ mod tests { insta::assert_snapshot!(plan, @r" CoalescePartitionsExec: fetch=3 FilterExec: pickup_date@0 = 2024-01-10, fetch=3 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------+ @@ -91,22 +89,21 @@ mod tests { let (plan, batches) = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) - .query("SELECT pickup_date FROM taxi LIMIT 3") + .build() + .await? + .query_batches("SELECT pickup_date FROM taxi LIMIT 3") .await?; insta::assert_snapshot!(plan, @r" - CooperativeExec + CoalescePartitionsExec: fetch=3 DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, fetch=3 "); - insta::assert_snapshot!(batches, @r" - +-------------+ - | pickup_date | - +-------------+ - | 2024-01-08 | - | 2024-01-08 | - | 2024-01-08 | - +-------------+ - "); + // No ORDER BY, so with several partitions any day's file can supply + // the rows; only their count is deterministic. + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 3 + ); Ok(()) } @@ -116,20 +113,24 @@ mod tests { let (_, explain) = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) + // With several partitions, how many open a file before the limit + // cancels the scan depends on scheduling. One partition stops after + // the first file, so the source's row count is deterministic. + .with_config_option("datafusion.execution.target_partitions", 1) // Only row counts, so the output doesn't vary between runs. .with_config_option("datafusion.explain.analyze_categories", "rows") .query("EXPLAIN ANALYZE SELECT pickup_date FROM taxi LIMIT 3") .await?; insta::assert_snapshot!(explain, @r" - +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ - | plan_type | plan | - +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ - | Plan with Metrics | CooperativeExec, metrics=[] | - | | DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, fetch=3, metrics=[output_rows=3, output_batches=1, batches_split=0] | - | | | - +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ - "); + +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ + | plan_type | plan | + +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ + | Plan with Metrics | CooperativeExec, metrics=[] | + | | DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, fetch=3, metrics=[output_rows=3, output_batches=1, batches_split=0] | + | | | + +-------------------+------------------------------------------------------------------------------------------------------------------------------------------------+ + "); Ok(()) } @@ -144,6 +145,7 @@ mod tests { )), ) .with_table(create_taxi_table().await?) + .with_config_option("datafusion.execution.target_partitions", 2) .build() .await?; harness.query_batches("SELECT * FROM taxi").await?; @@ -151,8 +153,8 @@ mod tests { harness.query_batches("SELECT * FROM taxi LIMIT 3").await?; let limited_scan_reads = storage.take_reads(); - // The exact counts depend on how many data files iceberg opens at - // once, which follows the runner's CPU count, so compare the two scans. + // Two partitions can open fewer than the seven files before LIMIT + // cancels the scan. Compare totals, not scheduler-dependent exact counts. assert!( limited_scan_reads < full_scan_reads, "a scan with LIMIT 3 read {limited_scan_reads} Parquet ranges, \ @@ -201,8 +203,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@2 ASC NULLS LAST], fetch=5 SortExec: TopK(fetch=5), expr=[pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@2 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@3 = 2024-01-10, projection=[vendor_id@0, pickup_at@1, pickup_location_id@2] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+ diff --git a/crates/datafusion/tests/plan_nodes.rs b/crates/datafusion/tests/plan_nodes.rs index 91b028a..c2c75a7 100644 --- a/crates/datafusion/tests/plan_nodes.rs +++ b/crates/datafusion/tests/plan_nodes.rs @@ -106,8 +106,10 @@ async fn run_source( source: &IcebergDataSource, ctx: &SessionContext, ) -> Result> { - let exec = DataSourceExec::from_data_source(source.clone()); - run(exec.as_ref(), ctx).await + // The lazy provider reads in one partition per target partition. + let exec = + CoalescePartitionsExec::new(DataSourceExec::from_data_source(source.clone())); + run(&exec, ctx).await } /// Returns the first node of type `T` in `plan`, depth first. diff --git a/crates/datafusion/tests/projection_pushdown.rs b/crates/datafusion/tests/projection_pushdown.rs index dc4f996..0e249e9 100644 --- a/crates/datafusion/tests/projection_pushdown.rs +++ b/crates/datafusion/tests/projection_pushdown.rs @@ -51,8 +51,7 @@ mod tests { RepartitionExec: partitioning=Hash([pickup_date@0], 16), input_partitions=16 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 >= 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date >= 2024-01-10 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date >= 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------+-------+ @@ -88,8 +87,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@2 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST], fetch=3 SortExec: TopK(fetch=3), expr=[pickup_at@2 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@3 = 2024-01-10, projection=[vendor_id@0, pickup_location_id@2, pickup_at@1] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+--------------------+ @@ -122,8 +120,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[max(taxi.trip_distance * taxi.fare_amount)] FilterExec: pickup_date@2 = 2024-01-10, projection=[trip_distance@0, fare_amount@1] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, fare_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[trip_distance, fare_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------------+ @@ -154,8 +151,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@5 ASC NULLS LAST], fetch=1 SortExec: TopK(fetch=1), expr=[pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@5 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@12 = 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, dropoff_at, passenger_count, trip_distance, pickup_location_id, dropoff_location_id, payment_type, fare_amount, tip_amount, tolls_amount, total_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, dropoff_at, passenger_count, trip_distance, pickup_location_id, dropoff_location_id, payment_type, fare_amount, tip_amount, tolls_amount, total_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+---------------------+-----------------+---------------+--------------------+---------------------+--------------+-------------+------------+--------------+--------------+-------------+ diff --git a/crates/datafusion/tests/schema_evolution.rs b/crates/datafusion/tests/schema_evolution.rs index bb865a5..3326641 100644 --- a/crates/datafusion/tests/schema_evolution.rs +++ b/crates/datafusion/tests/schema_evolution.rs @@ -24,6 +24,8 @@ use datafusion::arrow::compute::{concat_batches, sort_to_indices, take_record_ba use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::datasource::TableProvider; use datafusion::execution::context::SessionContext; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::common::collect; use iceberg::TableIdent; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; @@ -100,7 +102,7 @@ async fn test_scan_after_schema_evolution_reads_provider_columns() // The rows come from two data files, which may be read in either order. // They are joined under their own schema, not the plan's, so that a column // the plan does not report would show. - assert_eq!(plan.properties().partitioning.partition_count(), 1); + let plan = CoalescePartitionsExec::new(plan); let batches = collect(plan.execute(0, ctx.task_ctx())?).await?; let batch = concat_batches(&batches[0].schema(), &batches)?; let order = sort_to_indices(batch.column_by_name("foo1").unwrap(), None, None)?; diff --git a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt b/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt index bac4267..9636326 100644 --- a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt +++ b/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt @@ -51,7 +51,7 @@ logical_plan 01)Limit: skip=0, fetch=3 02)--TableScan: default.default.query_test_table projection=[id, name, score, category], fetch=3 physical_plan -01)CooperativeExec +01)CoalescePartitionsExec: fetch=3 02)--DataSourceExec: format=iceberg, projection=[id, name, score, category], provider=lazy, fetch=3 # Test SELECT * with ORDER BY and LIMIT diff --git a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt index 6a22dd9..205be33 100644 --- a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt @@ -38,7 +38,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = true # Query with is_active = true @@ -58,7 +58,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = false # Query with is_active = false @@ -77,7 +77,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = false # Query with is_active != true (includes false and NULL) diff --git a/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt index d25e445..611172f 100644 --- a/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt @@ -40,7 +40,7 @@ logical_plan 03)----TableScan: default.default.test_cast_table projection=[id, d], partial_filters=[CAST(default.default.test_cast_table.d AS Timestamp(ns)) < TimestampNanosecond(1672920000000000000, None)] physical_plan 01)FilterExec: CAST(d@1 AS Timestamp(ns)) < 1672920000000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, d], provider=lazy query I rowsort @@ -73,7 +73,7 @@ logical_plan 03)----TableScan: default.default.test_cast_table projection=[id, n], partial_filters=[CAST(default.default.test_cast_table.n AS Utf8) < Utf8("5")] physical_plan 01)FilterExec: CAST(n@1 AS Utf8) < 5, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, n], provider=lazy query I rowsort @@ -108,7 +108,7 @@ logical_plan 03)----TableScan: default.default.test_float_cast_table projection=[id, x], partial_filters=[CAST(default.default.test_float_cast_table.x AS Float64) > Float64(0.1)] physical_plan 01)FilterExec: CAST(x@1 AS Float64) > 0.1, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, x], provider=lazy query I rowsort @@ -137,7 +137,7 @@ logical_plan 03)----TableScan: default.default.test_isnan_cast_table projection=[id, s], partial_filters=[NOT isnan(CAST(default.default.test_isnan_cast_table.s AS Float64))] physical_plan 01)FilterExec: NOT isnan(CAST(s@1 AS Float64)), projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, s], provider=lazy query I rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt index dc4c36a..d3c93f6 100644 --- a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt @@ -36,7 +36,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name LIKE Utf8("Al%")] physical_plan 01)FilterExec: name@1 LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name STARTS WITH "Al" # Test LIKE filtering with case-sensitive match @@ -55,7 +55,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name NOT LIKE Utf8("Al%")] physical_plan 01)FilterExec: name@1 NOT LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name NOT STARTS WITH "Al" # Test NOT LIKE filtering @@ -129,7 +129,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name LIKE Utf8("Al\%%")] physical_plan 01)FilterExec: name@1 LIKE Al\%% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name STARTS WITH "Al%" query IT rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt index a23ad57..166368a 100644 --- a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt @@ -49,7 +49,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts = TimestampNanosecond(1672921800000000000, None)] physical_plan 01)FilterExec: ts@1 = 1672921800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts = 2023-01-05 12:30:00 # Verify timestamp equality filtering works @@ -67,7 +67,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts > TimestampNanosecond(1673308800000000000, None)] physical_plan 01)FilterExec: ts@1 > 1673308800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts > 2023-01-10 00:00:00 # Verify timestamp greater than filtering @@ -96,7 +96,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts >= TimestampNanosecond(1672876800000000000, None), default.default.test_timestamp_table.ts <= TimestampNanosecond(1673827199000000000, None)] physical_plan 01)FilterExec: ts@1 >= 1672876800000000000 AND ts@1 <= 1673827199000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59) # Test timestamp range predicate filtering @@ -161,7 +161,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_micros projection=[id, ts], partial_filters=[default.default.test_timestamp_micros.ts > TimestampMicrosecond(1672531200000000, None)] physical_plan 01)FilterExec: ts@1 > 1672531200000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts > 2023-01-01 00:00:00 query I? @@ -206,7 +206,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_us], partial_filters=[default.default.test_timestamptz.ts_us > TimestampMicrosecond(1672876800000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_us@1 > 1672876800000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_us], provider=lazy, predicate=ts_us > 2023-01-05 00:00:00 UTC query I @@ -223,7 +223,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_ns], partial_filters=[default.default.test_timestamptz.ts_ns > TimestampNanosecond(1672876800000000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_ns@1 > 1672876800000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_ns], provider=lazy, predicate=ts_ns > 2023-01-05 00:00:00 UTC query I @@ -240,7 +240,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_us], partial_filters=[default.default.test_timestamptz.ts_us > TimestampMicrosecond(1672876800000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_us@1 > 1672876800000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_us], provider=lazy, predicate=ts_us > 2023-01-05 00:00:00 UTC query I @@ -257,7 +257,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_naive], partial_filters=[default.default.test_timestamptz.ts_naive > TimestampNanosecond(1672876800000000000, None)] physical_plan 01)FilterExec: ts_naive@1 > 1672876800000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_naive], provider=lazy, predicate=ts_naive > 2023-01-05 00:00:00 query I