Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/datafusion/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ repository = { workspace = true }
autotests = false

[dependencies]
flume = "0.12"
async-trait = { workspace = true }
bytes = "1"
dashmap = "6"
Expand Down
3 changes: 2 additions & 1 deletion crates/datafusion/src/read/file_scan_task_provider/eager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use datafusion::common::stats::Precision;
use datafusion::common::{Result, Statistics, exec_err};
use datafusion::physical_plan::Partitioning;
use futures::StreamExt;
use futures::stream::BoxStream;
use iceberg::scan::FileScanTask;
use std::collections::BTreeSet;
use std::fmt::Formatter;
Expand Down Expand Up @@ -70,7 +71,7 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider {
self: Arc<Self>,
partition: usize,
_context: FileScanTaskProviderContext,
) -> Result<iceberg::scan::FileScanTaskStream> {
) -> Result<BoxStream<'static, Result<FileScanTask>>> {
Comment on lines -73 to +74

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue with FileScanTaskStream is that the errors returned in the messages are iceberg::Errors, not DataFusion errors. I did not notice in our past work, but the API here should probably be full DataFusion.

let Some(count) = self.groups().get(partition).map(Vec::len) else {
return exec_err!("Invalid Iceberg scan partition {partition}");
};
Expand Down
119 changes: 99 additions & 20 deletions crates/datafusion/src/read/file_scan_task_provider/lazy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,24 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::any::Any;
use std::fmt::Formatter;
use std::ops::DerefMut;
use std::sync::Arc;

use datafusion::arrow::datatypes::{Schema, SchemaRef};
use datafusion::common::runtime::SpawnedTask;
use datafusion::common::stats::Precision;
use datafusion::common::{Result, Statistics, exec_err};
use datafusion::common::{
DataFusionError, Result, Statistics, exec_datafusion_err, exec_err,
};
use datafusion::execution::TaskContext;
use datafusion::physical_plan::Partitioning;
use datafusion::prelude::Expr;
use futures::TryStreamExt;
use futures::stream::BoxStream;
use futures::{Stream, StreamExt, TryStreamExt};
use iceberg::expr::Predicate;
use iceberg::scan::TableScan;
use iceberg::scan::{FileScanTask, FileScanTaskStream, TableScan};
use iceberg::spec::SnapshotRef;
use iceberg::table::Table;

Expand All @@ -48,6 +54,51 @@ pub struct LazyFileScanTaskProvider {
snapshot_id: Option<i64>,
projection: Vec<String>,
predicate: Option<Predicate>,
partitions: usize,
}

#[derive(Debug, Default)]
enum StreamState {
#[default]
NotStarted,
Errored(Arc<DataFusionError>),
PartitionedFanout(PartitionedFanout),
}

#[derive(Debug)]
struct PartitionedFanout {
task: Arc<SpawnedTask<()>>,
per_partition_feed: Vec<flume::Receiver<Result<FileScanTask>>>,
}

impl PartitionedFanout {
fn new(partitions: usize, mut stream: FileScanTaskStream) -> Self {
let (tx, rx) = flume::unbounded();
let task = SpawnedTask::spawn(async move {
while let Some(result) = stream.next().await {
// Every partition dropped its feed, e.g. under LIMIT or when the
// query was cancelled, so stop discovering files.
if tx.send(result.map_err(to_datafusion_error)).is_err() {
break;
}
}
});
Self {
task: Arc::new(task),
per_partition_feed: vec![rx; partitions],
}
}

fn pop_stream(&mut self) -> Result<impl Stream<Item = Result<FileScanTask>> + use<>> {
let Some(feed) = self.per_partition_feed.pop() else {
return exec_err!("Requested more FileScanTask streams than available");
};
let task = Arc::clone(&self.task);
Ok(feed.into_stream().inspect(move |_| {
// Keep the task alive as long as the stream is in scope.
let _ = &task;
}))
}
}

impl LazyFileScanTaskProvider {
Expand All @@ -65,12 +116,8 @@ impl LazyFileScanTaskProvider {
"Iceberg eager planning requires at least one target partition"
);
}
let mut tasks = Arc::new(self).open(
0,
FileScanTaskProviderContext {
task_context: &context,
},
)?;
let mut tasks = self.scan.plan_files().await.map_err(to_datafusion_error)?;

let mut groups = Vec::new();
let mut partition = 0;
while let Some(task) = tasks.try_next().await.map_err(to_datafusion_error)? {
Expand Down Expand Up @@ -232,6 +279,7 @@ impl FileScanTaskProviderBuilder {
projection,
snapshot_id: self.snapshot_id,
predicate: self.predicate,
partitions: self.context.session_config().target_partitions(),
};
let provider: Arc<dyn FileScanTaskProvider> =
if IcebergDataFusionConfig::from_config_or_default(
Expand All @@ -250,22 +298,53 @@ impl FileScanTaskProviderBuilder {

impl FileScanTaskProvider for LazyFileScanTaskProvider {
fn partitioning(&self) -> Partitioning {
Partitioning::UnknownPartitioning(1)
Partitioning::UnknownPartitioning(self.partitions)
}

fn create_sibling_state(&self) -> Option<Arc<dyn Any + Send + Sync>> {
Some(Arc::new(tokio::sync::Mutex::<StreamState>::default()))
}

fn open(
self: Arc<Self>,
partition: usize,
_ctx: FileScanTaskProviderContext,
) -> Result<iceberg::scan::FileScanTaskStream> {
if partition != 0 {
return exec_err!("Invalid Iceberg scan partition {partition}");
}
_partition: usize,
ctx: FileScanTaskProviderContext,
) -> Result<BoxStream<'static, Result<FileScanTask>>> {
let scan = Arc::clone(&self.scan);
// Files are discovered when the stream is first polled, not here.
Ok(Box::pin(
futures::stream::once(async move { scan.plan_files().await }).try_flatten(),
))
let partitions = self.partitions;
let state = ctx
.sibling_state
.and_then(|state| state.downcast::<tokio::sync::Mutex<StreamState>>().ok())
.ok_or_else(|| {
exec_datafusion_err!("Missing sibling state for LazyFileScanTaskProvider")
})?;

// Files are discovered when the first stream is first polled, not here.
let stream = async move {
let mut state = state.lock().await;
match state.deref_mut() {
StreamState::NotStarted => {
let file_scan_task_stream = match scan.plan_files().await {
Ok(stream) => stream,
Err(err) => {
let err = Arc::new(to_datafusion_error(err));
*state = StreamState::Errored(Arc::clone(&err));
return Err(DataFusionError::Shared(err));
}
};
let mut partitioned_fanout =
PartitionedFanout::new(partitions, file_scan_task_stream);
let file_scan_task_stream = partitioned_fanout.pop_stream()?;
*state = StreamState::PartitionedFanout(partitioned_fanout);
Ok(file_scan_task_stream)
}
StreamState::Errored(err) => {
Err(DataFusionError::Shared(Arc::clone(err)))
}
StreamState::PartitionedFanout(fanout) => fanout.pop_stream(),
}
};
Ok(futures::stream::once(stream).try_flatten().boxed())
}

/// Estimates the rows the scan returns before its predicate and fetch,
Expand Down
15 changes: 14 additions & 1 deletion crates/datafusion/src/read/file_scan_task_provider/provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ use datafusion::arrow::datatypes::Schema;
use datafusion::common::{Result, Statistics};
use datafusion::execution::TaskContext;
use datafusion::physical_plan::Partitioning;
use futures::stream::BoxStream;
use iceberg::scan::FileScanTask;

/// Supplies the file tasks an [`IcebergDataSource`](crate::read::IcebergDataSource)
/// reads, split into output partitions.
Expand All @@ -45,13 +47,21 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync {
/// projected schema.
fn partitioning(&self) -> Partitioning;

/// Creates state shared by the partitions of one execution, passed to
/// [`Self::open`] through [`FileScanTaskProviderContext::sibling_state`].
/// The data source creates it once per execution, so re-executing a plan
/// after resetting its state starts afresh. Returns `None` by default.
fn create_sibling_state(&self) -> Option<Arc<dyn Any + Send + Sync>> {
None
}

/// Streams the file tasks of `partition`, which is below the
/// [`Self::partitioning`] partition count.
fn open(
self: Arc<Self>,
partition: usize,
context: FileScanTaskProviderContext,
) -> Result<iceberg::scan::FileScanTaskStream>;
) -> Result<BoxStream<'static, Result<FileScanTask>>>;

/// Statistics before the source's fetch, without discovering or reading files.
/// `None` requests the whole assignment; `Some(i)` requests partition `i`,
Expand Down Expand Up @@ -85,4 +95,7 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync {
/// table, belong to the provider rather than the data source.
pub struct FileScanTaskProviderContext<'a> {
pub task_context: &'a Arc<TaskContext>,
/// The state from [`FileScanTaskProvider::create_sibling_state`] for this
/// execution, or `None` when the caller opens partitions on their own.
pub sibling_state: Option<Arc<dyn Any + Send + Sync>>,
}
35 changes: 29 additions & 6 deletions crates/datafusion/src/read/scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,19 @@
// specific language governing permissions and limitations
// under the License.

use std::any::Any;
use std::fmt::{Debug, Formatter};
use std::sync::Arc;

use super::file_scan_task_provider::{FileScanTaskProvider, FileScanTaskProviderContext};
use crate::to_datafusion_error;
use crate::{from_datafusion_error, to_datafusion_error};

use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
use datafusion::common::stats::Precision;
use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::common::{Statistics, exec_err};
use datafusion::datasource::source::DataSource;
use datafusion::config::ConfigOptions;
use datafusion::datasource::source::{DataSource, OpenArgs};
use datafusion::error::Result;
use datafusion::execution::{SendableRecordBatchStream, TaskContext};
use datafusion::physical_expr::projection::ProjectionExprs;
Expand All @@ -35,7 +37,7 @@ use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSe
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{DisplayFormatType, Partitioning};
use datafusion::prelude::SessionConfig;
use futures::TryStreamExt;
use futures::{StreamExt, TryStreamExt};
use iceberg::arrow::{ArrowReader, ArrowReaderBuilder};
use iceberg::io::FileIO;

Expand Down Expand Up @@ -231,19 +233,36 @@ impl DataSource for IcebergDataSource {
partition: usize,
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
self.open_with_args(OpenArgs::new(partition, context))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is dropping support for direct DataSource::open on lazy scans intentional? I tried the same call on the parent and on this PR, with target_partitions = 1 so there’s no second reader to coordinate with:

let plan = harness.scan().await?;
let source = iceberg_source(&plan).unwrap();
let mut rows = source.open(0, harness.state().task_ctx())?;
assert!(rows.try_next().await?.is_some());

On the parent this returns a batch. On this PR, open returns Missing sibling state for LazyFileScanTaskProvider. The normal DataSourceExec path still works because it supplies sibling state; it’s the direct call that changed.

We should preserve the single-partition case, or, if direct opens are no longer supported for lazy scans, document that callers should execute the DataSourceExec.

}

fn create_sibling_state(
&self,
_config: &ConfigOptions,
) -> Option<Arc<dyn Any + Send + Sync>> {
self.file_scan_task_provider.create_sibling_state()
}

fn open_with_args(&self, args: OpenArgs) -> Result<SendableRecordBatchStream> {
let OpenArgs {
partition,
context,
sibling_state,
} = args;
if partition >= self.output_partitioning().partition_count() {
return exec_err!("Invalid Iceberg scan partition {partition}");
}
let tasks = Arc::clone(&self.file_scan_task_provider).open(
partition,
FileScanTaskProviderContext {
task_context: &context,
sibling_state,
},
)?;
let stream = self
.reader
.clone()
.read(tasks)
.read(tasks.map_err(from_datafusion_error).boxed())
.map_err(to_datafusion_error)?
.stream()
.map_err(to_datafusion_error);
Expand Down Expand Up @@ -344,7 +363,7 @@ mod tests {
use datafusion::common::Statistics;
use datafusion::common::stats::Precision;
use datafusion::datasource::source::DataSource;
use datafusion::prelude::SessionContext;
use datafusion::prelude::{SessionConfig, SessionContext};
use iceberg::io::FileIO;
use iceberg::spec::{
FormatVersion, MAIN_BRANCH, NestedField, Operation, PartitionSpec, PrimitiveType,
Expand Down Expand Up @@ -420,7 +439,11 @@ mod tests {
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, true),
]));
let context = SessionContext::new();
// One partition, so the fetch caps the whole scan rather than each of
// a machine-dependent number of partitions.
let context = SessionContext::new_with_config(
SessionConfig::new().with_target_partitions(1),
);
let (schema, provider) =
FileScanTaskProviderBuilder::new(context.task_ctx(), table.clone(), schema)
.with_projection(Some(&vec![0]))
Expand Down
Loading