Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 23 additions & 42 deletions crates/datafusion/src/read/file_scan_task_provider/lazy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,13 +119,11 @@ impl LazyFileScanTaskProvider {
}
}

/// Builds a task provider and its projected schema from a table scan request.
/// Builds a task provider for the columns of a projected Arrow schema.
///
/// Configuration is supplied at construction. `iceberg.execution.eager_file_planning`
/// selects eager collection; otherwise discovery remains deferred until execution.
/// The projected schema is returned alongside the provider so callers never need
/// to recover it by downcasting an eager or lazy implementation. Reader settings
/// and fetch belong to the data-source builder.
/// Pass the same projected schema to [`crate::read::IcebergDataSource::builder`].
///
/// ```
/// # use std::sync::Arc;
Expand All @@ -135,12 +133,10 @@ impl LazyFileScanTaskProvider {
/// async fn plan_tasks(
/// context: Arc<TaskContext>,
/// table: Table,
/// schema: SchemaRef,
/// projection: Option<&Vec<usize>>,
/// projected_schema: SchemaRef,
/// filters: &[Expr],
/// ) -> Result<(SchemaRef, Arc<dyn FileScanTaskProvider>)> {
/// FileScanTaskProviderBuilder::new(context, table, schema)
/// .with_projection(projection)
/// ) -> Result<Arc<dyn FileScanTaskProvider>> {
/// FileScanTaskProviderBuilder::new(context, table, projected_schema)
/// .with_filters(filters)
/// .build()
/// .await
Expand All @@ -150,34 +146,27 @@ impl LazyFileScanTaskProvider {
pub struct FileScanTaskProviderBuilder {
context: Arc<TaskContext>,
table: Table,
schema: SchemaRef,
projection: Option<Vec<usize>>,
projected_schema: SchemaRef,
snapshot_id: Option<i64>,
predicate: Option<Predicate>,
}

impl FileScanTaskProviderBuilder {
/// Starts with the planning context and the schema advertised by the table
/// provider. Projection indices refer to this schema, not the table's current
/// schema. Pass an already-projected schema and omit projection to retain it.
pub fn new(context: Arc<TaskContext>, table: Table, schema: SchemaRef) -> Self {
/// Selects the columns named in `projected_schema`, in schema order.
pub fn new(
context: Arc<TaskContext>,
table: Table,
projected_schema: SchemaRef,
) -> Self {
Self {
context,
table,
schema,
projection: None,
projected_schema,
snapshot_id: None,
predicate: None,
}
}

/// Selects columns from the supplied schema in this order, or all its columns
/// for `None`. Indices are validated when [`Self::build`] is called.
pub fn with_projection(mut self, projection: Option<&Vec<usize>>) -> Self {
self.projection = projection.cloned();
self
}

/// Selects a snapshot, or the retained table's current snapshot for `None`.
/// The supplied schema is not replaced with the snapshot's schema.
pub fn with_snapshot_id(mut self, snapshot_id: Option<i64>) -> Self {
Expand All @@ -199,18 +188,14 @@ impl FileScanTaskProviderBuilder {

/// Resolves the snapshot, validates projection and builds the configured provider.
/// Lazy mode performs no file discovery; eager mode awaits [`LazyFileScanTaskProvider::into_eager`].
/// The returned schema matches the columns selected by both implementations.
///
/// # Errors
///
/// Returns an error if the snapshot does not exist, projection or predicate
/// binding fails, or eager file discovery fails.
pub async fn build(self) -> Result<(SchemaRef, Arc<dyn FileScanTaskProvider>)> {
let schema = match self.projection {
Some(projection) => Arc::new(self.schema.project(&projection)?),
None => self.schema,
};
let projection: Vec<String> = schema
pub async fn build(self) -> Result<Arc<dyn FileScanTaskProvider>> {
let projection: Vec<String> = self
.projected_schema
.fields()
.iter()
.map(|field| field.name().clone())
Expand All @@ -228,23 +213,19 @@ impl FileScanTaskProviderBuilder {
let discovery = LazyFileScanTaskProvider {
table: self.table,
scan,
schema: Arc::clone(&schema),
schema: self.projected_schema,
projection,
snapshot_id: self.snapshot_id,
predicate: self.predicate,
};
let provider: Arc<dyn FileScanTaskProvider> =
if IcebergDataFusionConfig::from_config_or_default(
self.context.session_config(),
)
if IcebergDataFusionConfig::from_config_or_default(self.context.session_config())
.execution
.eager_file_planning
{
Arc::new(discovery.into_eager(self.context).await?)
} else {
Arc::new(discovery)
};
Ok((schema, provider))
{
Ok(Arc::new(discovery.into_eager(self.context).await?))
} else {
Ok(Arc::new(discovery))
}
}
}

Expand Down
38 changes: 22 additions & 16 deletions crates/datafusion/src/read/scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -416,18 +416,21 @@ mod tests {
snapshot_id: Option<i64>,
fetch: Option<usize>,
) -> Statistics {
let schema = Arc::new(ArrowSchema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, true),
]));
let schema = Arc::new(ArrowSchema::new(vec![Field::new(
"id",
DataType::Int64,
false,
)]));
let context = SessionContext::new();
let (schema, provider) =
FileScanTaskProviderBuilder::new(context.task_ctx(), table.clone(), schema)
.with_projection(Some(&vec![0]))
.with_snapshot_id(snapshot_id)
.build()
.await
.unwrap();
let provider = FileScanTaskProviderBuilder::new(
context.task_ctx(),
table.clone(),
Arc::clone(&schema),
)
.with_snapshot_id(snapshot_id)
.build()
.await
.unwrap();
let source = IcebergDataSource::builder(
&context.copied_config(),
schema,
Expand Down Expand Up @@ -541,11 +544,14 @@ mod tests {
Field::new("name", DataType::Utf8, true),
]));
let context = SessionContext::new();
let (schema, provider) =
FileScanTaskProviderBuilder::new(context.task_ctx(), table.clone(), schema)
.build()
.await
.unwrap();
let provider = FileScanTaskProviderBuilder::new(
context.task_ctx(),
table.clone(),
Arc::clone(&schema),
)
.build()
.await
.unwrap();
let source = IcebergDataSource::builder(
&context.copied_config(),
schema,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
use datafusion::catalog::Session;
use datafusion::common::utils::project_schema;
use datafusion::common::{config_datafusion_err, not_impl_err};
use datafusion::datasource::source::DataSourceExec;
use datafusion::datasource::{TableProvider, TableType};
Expand Down Expand Up @@ -139,12 +140,12 @@ impl TableProvider for IcebergCatalogTableProvider {
.map_err(to_datafusion_error)?;

// Create scan with fresh metadata (always use current snapshot).
let (schema, tasks) = FileScanTaskProviderBuilder::new(
let schema = project_schema(&self.schema, projection)?;
let tasks = FileScanTaskProviderBuilder::new(
state.task_ctx(),
table.clone(),
self.schema.clone(),
Arc::clone(&schema),
)
.with_projection(projection)
.with_filters(filters)
.build()
.await?;
Expand Down
7 changes: 4 additions & 3 deletions crates/datafusion/src/table_provider/static_table_provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
use datafusion::catalog::Session;
use datafusion::common::utils::project_schema;
use datafusion::common::{exec_datafusion_err, not_impl_err};
use datafusion::datasource::source::DataSourceExec;
use datafusion::datasource::{TableProvider, TableType};
Expand Down Expand Up @@ -123,12 +124,12 @@ impl TableProvider for IcebergStaticTableProvider {
limit: Option<usize>,
) -> Result<Arc<dyn ExecutionPlan>> {
// Use cached table (no refresh).
let (schema, tasks) = FileScanTaskProviderBuilder::new(
let schema = project_schema(&self.schema, projection)?;
let tasks = FileScanTaskProviderBuilder::new(
state.task_ctx(),
self.table.clone(),
self.schema.clone(),
Arc::clone(&schema),
)
.with_projection(projection)
.with_snapshot_id(self.snapshot_id)
.with_filters(filters)
.build()
Expand Down
10 changes: 5 additions & 5 deletions crates/datafusion/tests/plan_nodes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ fn lazy_files(scan: &IcebergDataSource) -> &LazyFileScanTaskProvider {
async fn rebuild_scan(scan: &IcebergDataSource) -> IcebergDataSource {
let files = lazy_files(scan);
let context = SessionContext::new();
let (schema, provider) = FileScanTaskProviderBuilder::new(
let provider = FileScanTaskProviderBuilder::new(
context.task_ctx(),
files.table().clone(),
files.schema(),
Expand All @@ -91,7 +91,7 @@ async fn rebuild_scan(scan: &IcebergDataSource) -> IcebergDataSource {
.unwrap();
IcebergDataSource::builder(
&context.copied_config(),
schema,
scan.schema(),
scan.file_io().clone(),
files.table().reader_builder(),
provider,
Expand Down Expand Up @@ -260,17 +260,17 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box<dyn Error>> {
// A scan reads the columns of its schema and no others, so one built over
// part of the table returns only those columns, from the pinned snapshot.
let foo2_only = Arc::new(pinned.schema().project(&[1])?);
let (schema, files) = FileScanTaskProviderBuilder::new(
let files = FileScanTaskProviderBuilder::new(
ctx.task_ctx(),
pinned.table().clone(),
foo2_only,
Arc::clone(&foo2_only),
)
.with_snapshot_id(Some(snapshot_id))
.build()
.await?;
let partial = IcebergDataSource::builder(
&ctx.copied_config(),
schema,
foo2_only,
pinned.table().file_io().clone(),
pinned.table().reader_builder(),
files,
Expand Down