diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index 2a5b189..6843f56 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -119,13 +119,11 @@ impl LazyFileScanTaskProvider { } } -/// Builds a task provider and its projected schema from a table scan request. +/// Builds a task provider for the columns of a projected Arrow schema. /// /// Configuration is supplied at construction. `iceberg.execution.eager_file_planning` /// selects eager collection; otherwise discovery remains deferred until execution. -/// The projected schema is returned alongside the provider so callers never need -/// to recover it by downcasting an eager or lazy implementation. Reader settings -/// and fetch belong to the data-source builder. +/// Pass the same projected schema to [`crate::read::IcebergDataSource::builder`]. /// /// ``` /// # use std::sync::Arc; @@ -135,12 +133,10 @@ impl LazyFileScanTaskProvider { /// async fn plan_tasks( /// context: Arc, /// table: Table, -/// schema: SchemaRef, -/// projection: Option<&Vec>, +/// projected_schema: SchemaRef, /// filters: &[Expr], -/// ) -> Result<(SchemaRef, Arc)> { -/// FileScanTaskProviderBuilder::new(context, table, schema) -/// .with_projection(projection) +/// ) -> Result> { +/// FileScanTaskProviderBuilder::new(context, table, projected_schema) /// .with_filters(filters) /// .build() /// .await @@ -150,34 +146,27 @@ impl LazyFileScanTaskProvider { pub struct FileScanTaskProviderBuilder { context: Arc, table: Table, - schema: SchemaRef, - projection: Option>, + projected_schema: SchemaRef, snapshot_id: Option, predicate: Option, } impl FileScanTaskProviderBuilder { - /// Starts with the planning context and the schema advertised by the table - /// provider. Projection indices refer to this schema, not the table's current - /// schema. Pass an already-projected schema and omit projection to retain it. - pub fn new(context: Arc, table: Table, schema: SchemaRef) -> Self { + /// Selects the columns named in `projected_schema`, in schema order. + pub fn new( + context: Arc, + table: Table, + projected_schema: SchemaRef, + ) -> Self { Self { context, table, - schema, - projection: None, + projected_schema, snapshot_id: None, predicate: None, } } - /// Selects columns from the supplied schema in this order, or all its columns - /// for `None`. Indices are validated when [`Self::build`] is called. - pub fn with_projection(mut self, projection: Option<&Vec>) -> Self { - self.projection = projection.cloned(); - self - } - /// Selects a snapshot, or the retained table's current snapshot for `None`. /// The supplied schema is not replaced with the snapshot's schema. pub fn with_snapshot_id(mut self, snapshot_id: Option) -> Self { @@ -199,18 +188,14 @@ impl FileScanTaskProviderBuilder { /// Resolves the snapshot, validates projection and builds the configured provider. /// Lazy mode performs no file discovery; eager mode awaits [`LazyFileScanTaskProvider::into_eager`]. - /// The returned schema matches the columns selected by both implementations. /// /// # Errors /// /// Returns an error if the snapshot does not exist, projection or predicate /// binding fails, or eager file discovery fails. - pub async fn build(self) -> Result<(SchemaRef, Arc)> { - let schema = match self.projection { - Some(projection) => Arc::new(self.schema.project(&projection)?), - None => self.schema, - }; - let projection: Vec = schema + pub async fn build(self) -> Result> { + let projection: Vec = self + .projected_schema .fields() .iter() .map(|field| field.name().clone()) @@ -228,23 +213,19 @@ impl FileScanTaskProviderBuilder { let discovery = LazyFileScanTaskProvider { table: self.table, scan, - schema: Arc::clone(&schema), + schema: self.projected_schema, projection, snapshot_id: self.snapshot_id, predicate: self.predicate, }; - let provider: Arc = - if IcebergDataFusionConfig::from_config_or_default( - self.context.session_config(), - ) + if IcebergDataFusionConfig::from_config_or_default(self.context.session_config()) .execution .eager_file_planning - { - Arc::new(discovery.into_eager(self.context).await?) - } else { - Arc::new(discovery) - }; - Ok((schema, provider)) + { + Ok(Arc::new(discovery.into_eager(self.context).await?)) + } else { + Ok(Arc::new(discovery)) + } } } diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index f74e8d1..30bb423 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -416,18 +416,21 @@ mod tests { snapshot_id: Option, fetch: Option, ) -> Statistics { - let schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("name", DataType::Utf8, true), - ])); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + DataType::Int64, + false, + )])); let context = SessionContext::new(); - let (schema, provider) = - FileScanTaskProviderBuilder::new(context.task_ctx(), table.clone(), schema) - .with_projection(Some(&vec![0])) - .with_snapshot_id(snapshot_id) - .build() - .await - .unwrap(); + let provider = FileScanTaskProviderBuilder::new( + context.task_ctx(), + table.clone(), + Arc::clone(&schema), + ) + .with_snapshot_id(snapshot_id) + .build() + .await + .unwrap(); let source = IcebergDataSource::builder( &context.copied_config(), schema, @@ -541,11 +544,14 @@ mod tests { Field::new("name", DataType::Utf8, true), ])); let context = SessionContext::new(); - let (schema, provider) = - FileScanTaskProviderBuilder::new(context.task_ctx(), table.clone(), schema) - .build() - .await - .unwrap(); + let provider = FileScanTaskProviderBuilder::new( + context.task_ctx(), + table.clone(), + Arc::clone(&schema), + ) + .build() + .await + .unwrap(); let source = IcebergDataSource::builder( &context.copied_config(), schema, diff --git a/crates/datafusion/src/table_provider/catalog_table_provider.rs b/crates/datafusion/src/table_provider/catalog_table_provider.rs index 70fd7d8..90eb52b 100644 --- a/crates/datafusion/src/table_provider/catalog_table_provider.rs +++ b/crates/datafusion/src/table_provider/catalog_table_provider.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::Session; +use datafusion::common::utils::project_schema; use datafusion::common::{config_datafusion_err, not_impl_err}; use datafusion::datasource::source::DataSourceExec; use datafusion::datasource::{TableProvider, TableType}; @@ -139,12 +140,12 @@ impl TableProvider for IcebergCatalogTableProvider { .map_err(to_datafusion_error)?; // Create scan with fresh metadata (always use current snapshot). - let (schema, tasks) = FileScanTaskProviderBuilder::new( + let schema = project_schema(&self.schema, projection)?; + let tasks = FileScanTaskProviderBuilder::new( state.task_ctx(), table.clone(), - self.schema.clone(), + Arc::clone(&schema), ) - .with_projection(projection) .with_filters(filters) .build() .await?; diff --git a/crates/datafusion/src/table_provider/static_table_provider.rs b/crates/datafusion/src/table_provider/static_table_provider.rs index 06205f7..0052f82 100644 --- a/crates/datafusion/src/table_provider/static_table_provider.rs +++ b/crates/datafusion/src/table_provider/static_table_provider.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::Session; +use datafusion::common::utils::project_schema; use datafusion::common::{exec_datafusion_err, not_impl_err}; use datafusion::datasource::source::DataSourceExec; use datafusion::datasource::{TableProvider, TableType}; @@ -123,12 +124,12 @@ impl TableProvider for IcebergStaticTableProvider { limit: Option, ) -> Result> { // Use cached table (no refresh). - let (schema, tasks) = FileScanTaskProviderBuilder::new( + let schema = project_schema(&self.schema, projection)?; + let tasks = FileScanTaskProviderBuilder::new( state.task_ctx(), self.table.clone(), - self.schema.clone(), + Arc::clone(&schema), ) - .with_projection(projection) .with_snapshot_id(self.snapshot_id) .with_filters(filters) .build() diff --git a/crates/datafusion/tests/plan_nodes.rs b/crates/datafusion/tests/plan_nodes.rs index 91b028a..f5d469d 100644 --- a/crates/datafusion/tests/plan_nodes.rs +++ b/crates/datafusion/tests/plan_nodes.rs @@ -79,7 +79,7 @@ fn lazy_files(scan: &IcebergDataSource) -> &LazyFileScanTaskProvider { async fn rebuild_scan(scan: &IcebergDataSource) -> IcebergDataSource { let files = lazy_files(scan); let context = SessionContext::new(); - let (schema, provider) = FileScanTaskProviderBuilder::new( + let provider = FileScanTaskProviderBuilder::new( context.task_ctx(), files.table().clone(), files.schema(), @@ -91,7 +91,7 @@ async fn rebuild_scan(scan: &IcebergDataSource) -> IcebergDataSource { .unwrap(); IcebergDataSource::builder( &context.copied_config(), - schema, + scan.schema(), scan.file_io().clone(), files.table().reader_builder(), provider, @@ -260,17 +260,17 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { // A scan reads the columns of its schema and no others, so one built over // part of the table returns only those columns, from the pinned snapshot. let foo2_only = Arc::new(pinned.schema().project(&[1])?); - let (schema, files) = FileScanTaskProviderBuilder::new( + let files = FileScanTaskProviderBuilder::new( ctx.task_ctx(), pinned.table().clone(), - foo2_only, + Arc::clone(&foo2_only), ) .with_snapshot_id(Some(snapshot_id)) .build() .await?; let partial = IcebergDataSource::builder( &ctx.copied_config(), - schema, + foo2_only, pinned.table().file_io().clone(), pinned.table().reader_builder(), files,