Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crates/datafusion/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,15 @@ futures = "0.3"
iceberg = { workspace = true }
parquet = "59.2"
tokio = { workspace = true }
tracing = { workspace = true }
uuid = { version = "1.18", features = ["v7"] }

[dev-dependencies]
iceberg-datafusion-test-tools = { workspace = true }
insta = "1"
parquet = "59.2"
tempfile = "3.18"
tracing-subscriber = "0.3"

# Every integration test is a module of `tests/main.rs`, so they all link into one binary
# instead of one binary per file.
Expand Down
9 changes: 9 additions & 0 deletions crates/datafusion/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,12 @@
# Apache Iceberg DataFusion Integration

This crate contains the integration of Apache DataFusion and Apache Iceberg.

### Tracing file discovery

With an INFO-level `tracing` subscriber, a scan emits `iceberg.plan_scan` while
building the task provider. With `iceberg.execution.eager_file_planning` enabled,
`iceberg.plan_files` is a child of that span. Otherwise it appears during execution,
when the reader opens the lazy file-task stream. Its lifetime can include time
between polls; it is not a measure of S3 request latency. On exhaustion it records
`file_tasks`, the number of successfully discovered tasks.
62 changes: 44 additions & 18 deletions crates/datafusion/src/read/file_scan_task_provider/lazy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,20 @@
// under the License.
use std::fmt::Formatter;
use std::sync::Arc;
use std::task::Poll;

use datafusion::arrow::datatypes::{Schema, SchemaRef};
use datafusion::common::stats::Precision;
use datafusion::common::{Result, Statistics, exec_err};
use datafusion::execution::TaskContext;
use datafusion::physical_plan::Partitioning;
use datafusion::prelude::Expr;
use futures::TryStreamExt;
use futures::{TryStreamExt, stream};
use iceberg::expr::Predicate;
use iceberg::scan::TableScan;
use iceberg::scan::{FileScanTaskStream, TableScan};
use iceberg::spec::SnapshotRef;
use iceberg::table::Table;
use tracing::{Instrument, field, info_span};

use super::eager::EagerFileScanTaskProvider;
use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext};
Expand Down Expand Up @@ -65,12 +67,7 @@ impl LazyFileScanTaskProvider {
"Iceberg eager planning requires at least one target partition"
);
}
let mut tasks = Arc::new(self).open(
0,
FileScanTaskProviderContext {
task_context: &context,
},
)?;
let mut tasks = discover_files(self.scan, true);
let mut groups = Vec::new();
let mut partition = 0;
while let Some(task) = tasks.try_next().await.map_err(to_datafusion_error)? {
Expand Down Expand Up @@ -194,6 +191,16 @@ impl FileScanTaskProviderBuilder {
/// Returns an error if the snapshot does not exist, projection or predicate
/// binding fails, or eager file discovery fails.
pub async fn build(self) -> Result<Arc<dyn FileScanTaskProvider>> {
let eager = IcebergDataFusionConfig::from_config_or_default(
self.context.session_config(),
)
.execution
.eager_file_planning;
self.build_provider(eager).await
}

#[tracing::instrument(name = "iceberg.plan_scan", skip(self))]
async fn build_provider(self, eager: bool) -> Result<Arc<dyn FileScanTaskProvider>> {
let projection: Vec<String> = self
.projected_schema
.fields()
Expand All @@ -218,17 +225,40 @@ impl FileScanTaskProviderBuilder {
snapshot_id: self.snapshot_id,
predicate: self.predicate,
};
if IcebergDataFusionConfig::from_config_or_default(self.context.session_config())
.execution
.eager_file_planning
{
if eager {
Ok(Arc::new(discovery.into_eager(self.context).await?))
} else {
Ok(Arc::new(discovery))
}
}
}

/// Capture the caller's span before handing the stream to the reader. File discovery
/// remains deferred until the first poll.
fn discover_files(scan: Arc<TableScan>, eager: bool) -> FileScanTaskStream {
let span = info_span!("iceberg.plan_files", eager, file_tasks = field::Empty);
Box::pin(
stream::once(async move {
let mut tasks = scan.plan_files().instrument(span.clone()).await?;
let mut file_tasks = 0usize;
Ok::<_, iceberg::Error>(stream::poll_fn(move |cx| {
span.in_scope(|| {
let next = tasks.as_mut().poll_next(cx);
match &next {
Poll::Ready(Some(Ok(_))) => file_tasks += 1,
Poll::Ready(None) => {
span.record("file_tasks", file_tasks);
}
_ => {}
}
next
})
}))
})
.try_flatten(),
)
}

impl FileScanTaskProvider for LazyFileScanTaskProvider {
fn partitioning(&self) -> Partitioning {
Partitioning::UnknownPartitioning(1)
Expand All @@ -238,15 +268,11 @@ impl FileScanTaskProvider for LazyFileScanTaskProvider {
self: Arc<Self>,
partition: usize,
_ctx: FileScanTaskProviderContext,
) -> Result<iceberg::scan::FileScanTaskStream> {
) -> Result<FileScanTaskStream> {
if partition != 0 {
return exec_err!("Invalid Iceberg scan partition {partition}");
}
let scan = Arc::clone(&self.scan);
// Files are discovered when the stream is first polled, not here.
Ok(Box::pin(
futures::stream::once(async move { scan.plan_files().await }).try_flatten(),
))
Ok(discover_files(Arc::clone(&self.scan), false))
}

/// Estimates the rows the scan returns before its predicate and fetch,
Expand Down
1 change: 1 addition & 0 deletions crates/datafusion/tests/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,6 @@ mod metadata_tables;
mod page_index;
mod plan_nodes;
mod projection_pushdown;
mod scan_tracing;
mod schema_evolution;
mod statistics;
87 changes: 87 additions & 0 deletions crates/datafusion/tests/scan_tracing.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

use std::fs::read_to_string;

use datafusion::common::Result;
use datafusion::physical_plan::collect;
use iceberg_datafusion::{IcebergDataFusionConfig, IcebergTableProviderFactory};
use iceberg_datafusion_test_tools::{IcebergTestHarness, create_taxi_table};
use tempfile::NamedTempFile;
use tracing::subscriber::DefaultGuard;
use tracing_subscriber::filter::filter_fn;
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::prelude::*;

#[tokio::test]
async fn eager_discovery_is_a_child_of_scan_planning() -> Result<()> {
let harness = IcebergTestHarness::builder()
.with_iceberg_table_factory(IcebergTableProviderFactory::new())
.with_table(create_taxi_table().await?)
.with_session_extension(|mut state| {
state
.config()
.get_or_insert_default()
.options_mut()
.extensions
.insert(IcebergDataFusionConfig::default());
state
})
.with_config_option("iceberg.execution.eager_file_planning", true)
.build()
.await?;
let (trace, _guard) = capture_planning_spans()?;
harness.scan().await?;
insta::assert_snapshot!(read_to_string(trace.path())?, @"
iceberg.plan_scan{eager=true}:iceberg.plan_files{eager=true file_tasks=7}: close
iceberg.plan_scan{eager=true}: close
");
Ok(())
}

#[tokio::test]
async fn lazy_discovery_is_deferred_until_execution() -> Result<()> {
let harness = IcebergTestHarness::builder()
.with_iceberg_table_factory(IcebergTableProviderFactory::new())
.with_table(create_taxi_table().await?)
.build()
.await?;
let (trace, _guard) = capture_planning_spans()?;
let plan = harness.scan().await?;
insta::assert_snapshot!(read_to_string(trace.path())?, @"iceberg.plan_scan{eager=false}: close");
collect(plan, harness.state().task_ctx()).await?;
insta::assert_snapshot!(read_to_string(trace.path())?, @"
iceberg.plan_scan{eager=false}: close
iceberg.plan_files{eager=false file_tasks=7}: close
");
Ok(())
}

fn capture_planning_spans() -> Result<(NamedTempFile, DefaultGuard)> {
let trace = NamedTempFile::new()?;
let layer = tracing_subscriber::fmt::layer()
.without_time()
.with_target(false)
.with_level(false)
.with_ansi(false)
.with_span_events(FmtSpan::CLOSE)
.with_writer(trace.reopen()?)
.with_filter(filter_fn(|m| m.name().starts_with("iceberg.plan_")));
let guard =
tracing::subscriber::set_default(tracing_subscriber::registry().with(layer));
Ok((trace, guard))
}