From 7c10c6953137c07bc97d712827592b39a26fb44f Mon Sep 17 00:00:00 2001 From: alexanderbianchi <75697973+alexanderbianchi@users.noreply.github.com> Date: Fri, 20 Nov 2026 23:59:27 +0000 Subject: [PATCH 1/2] feat: trace eager and lazy Iceberg file discovery --- Cargo.lock | 2 + crates/datafusion/Cargo.toml | 2 + crates/datafusion/README.md | 9 ++ .../src/read/file_scan_task_provider/lazy.rs | 62 +++++++++---- crates/datafusion/tests/main.rs | 1 + crates/datafusion/tests/scan_tracing.rs | 87 +++++++++++++++++++ 6 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 crates/datafusion/tests/scan_tracing.rs diff --git a/Cargo.lock b/Cargo.lock index b111a48..2e3b3c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3250,6 +3250,8 @@ dependencies = [ "parquet", "tempfile", "tokio", + "tracing", + "tracing-subscriber", "uuid", ] diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml index e7fb281..40b0342 100644 --- a/crates/datafusion/Cargo.toml +++ b/crates/datafusion/Cargo.toml @@ -38,6 +38,7 @@ futures = "0.3" iceberg = { workspace = true } parquet = "59.2" tokio = { workspace = true } +tracing = { workspace = true } uuid = { version = "1.18", features = ["v7"] } [dev-dependencies] @@ -45,6 +46,7 @@ iceberg-datafusion-test-tools = { workspace = true } insta = "1" parquet = "59.2" tempfile = "3.18" +tracing-subscriber = "0.3" # Every integration test is a module of `tests/main.rs`, so they all link into one binary # instead of one binary per file. diff --git a/crates/datafusion/README.md b/crates/datafusion/README.md index 134a8ef..f5c0e13 100644 --- a/crates/datafusion/README.md +++ b/crates/datafusion/README.md @@ -20,3 +20,12 @@ # Apache Iceberg DataFusion Integration This crate contains the integration of Apache DataFusion and Apache Iceberg. + +### Tracing file discovery + +With an INFO-level `tracing` subscriber, a scan emits `iceberg.plan_scan` while +building the task provider. With `iceberg.execution.eager_file_planning` enabled, +`iceberg.plan_files` is a child of that span. Otherwise it appears during execution, +when the reader opens the lazy file-task stream. Its lifetime can include time +between polls; it is not a measure of S3 request latency. On exhaustion it records +`file_tasks`, the number of successfully discovered tasks. diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index 6843f56..c8d1483 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -16,6 +16,7 @@ // under the License. use std::fmt::Formatter; use std::sync::Arc; +use std::task::Poll; use datafusion::arrow::datatypes::{Schema, SchemaRef}; use datafusion::common::stats::Precision; @@ -23,11 +24,12 @@ use datafusion::common::{Result, Statistics, exec_err}; use datafusion::execution::TaskContext; use datafusion::physical_plan::Partitioning; use datafusion::prelude::Expr; -use futures::TryStreamExt; +use futures::{TryStreamExt, stream}; use iceberg::expr::Predicate; -use iceberg::scan::TableScan; +use iceberg::scan::{FileScanTaskStream, TableScan}; use iceberg::spec::SnapshotRef; use iceberg::table::Table; +use tracing::{Instrument, field, info_span}; use super::eager::EagerFileScanTaskProvider; use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext}; @@ -65,12 +67,7 @@ impl LazyFileScanTaskProvider { "Iceberg eager planning requires at least one target partition" ); } - let mut tasks = Arc::new(self).open( - 0, - FileScanTaskProviderContext { - task_context: &context, - }, - )?; + let mut tasks = discover_files(self.scan, true); let mut groups = Vec::new(); let mut partition = 0; while let Some(task) = tasks.try_next().await.map_err(to_datafusion_error)? { @@ -194,6 +191,16 @@ impl FileScanTaskProviderBuilder { /// Returns an error if the snapshot does not exist, projection or predicate /// binding fails, or eager file discovery fails. pub async fn build(self) -> Result> { + let eager = IcebergDataFusionConfig::from_config_or_default( + self.context.session_config(), + ) + .execution + .eager_file_planning; + self.build_provider(eager).await + } + + #[tracing::instrument(name = "iceberg.plan_scan", skip(self))] + async fn build_provider(self, eager: bool) -> Result> { let projection: Vec = self .projected_schema .fields() @@ -218,10 +225,7 @@ impl FileScanTaskProviderBuilder { snapshot_id: self.snapshot_id, predicate: self.predicate, }; - if IcebergDataFusionConfig::from_config_or_default(self.context.session_config()) - .execution - .eager_file_planning - { + if eager { Ok(Arc::new(discovery.into_eager(self.context).await?)) } else { Ok(Arc::new(discovery)) @@ -229,6 +233,32 @@ impl FileScanTaskProviderBuilder { } } +/// Capture the caller's span before handing the stream to the reader. File discovery +/// remains deferred until the first poll. +fn discover_files(scan: Arc, eager: bool) -> FileScanTaskStream { + let span = info_span!("iceberg.plan_files", eager, file_tasks = field::Empty); + Box::pin( + stream::once(async move { + let mut tasks = scan.plan_files().instrument(span.clone()).await?; + let mut file_tasks = 0usize; + Ok(stream::poll_fn(move |cx| { + span.in_scope(|| { + let next = tasks.as_mut().poll_next(cx); + match &next { + Poll::Ready(Some(Ok(_))) => file_tasks += 1, + Poll::Ready(None) => { + span.record("file_tasks", file_tasks); + } + _ => {} + } + next + }) + })) + }) + .try_flatten(), + ) +} + impl FileScanTaskProvider for LazyFileScanTaskProvider { fn partitioning(&self) -> Partitioning { Partitioning::UnknownPartitioning(1) @@ -238,15 +268,11 @@ impl FileScanTaskProvider for LazyFileScanTaskProvider { self: Arc, partition: usize, _ctx: FileScanTaskProviderContext, - ) -> Result { + ) -> Result { if partition != 0 { return exec_err!("Invalid Iceberg scan partition {partition}"); } - let scan = Arc::clone(&self.scan); - // Files are discovered when the stream is first polled, not here. - Ok(Box::pin( - futures::stream::once(async move { scan.plan_files().await }).try_flatten(), - )) + Ok(discover_files(Arc::clone(&self.scan), false)) } /// Estimates the rows the scan returns before its predicate and fetch, diff --git a/crates/datafusion/tests/main.rs b/crates/datafusion/tests/main.rs index 263af20..9128c8a 100644 --- a/crates/datafusion/tests/main.rs +++ b/crates/datafusion/tests/main.rs @@ -26,5 +26,6 @@ mod metadata_tables; mod page_index; mod plan_nodes; mod projection_pushdown; +mod scan_tracing; mod schema_evolution; mod statistics; diff --git a/crates/datafusion/tests/scan_tracing.rs b/crates/datafusion/tests/scan_tracing.rs new file mode 100644 index 0000000..6994ad9 --- /dev/null +++ b/crates/datafusion/tests/scan_tracing.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fs::read_to_string; + +use datafusion::common::Result; +use datafusion::physical_plan::collect; +use iceberg_datafusion::{IcebergDataFusionConfig, IcebergTableProviderFactory}; +use iceberg_datafusion_test_tools::{IcebergTestHarness, create_taxi_table}; +use tempfile::NamedTempFile; +use tracing::subscriber::DefaultGuard; +use tracing_subscriber::filter::filter_fn; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::prelude::*; + +#[tokio::test] +async fn eager_discovery_is_a_child_of_scan_planning() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_table(create_taxi_table().await?) + .with_session_extension(|mut state| { + state + .config() + .get_or_insert_default() + .options_mut() + .extensions + .insert(IcebergDataFusionConfig::default()); + state + }) + .with_config_option("iceberg.execution.eager_file_planning", true) + .build() + .await?; + let (trace, _guard) = capture_planning_spans()?; + harness.scan().await?; + insta::assert_snapshot!(read_to_string(trace.path())?, @" + iceberg.plan_scan{eager=true}:iceberg.plan_files{eager=true file_tasks=7}: close + iceberg.plan_scan{eager=true}: close + "); + Ok(()) +} + +#[tokio::test] +async fn lazy_discovery_is_deferred_until_execution() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_table(create_taxi_table().await?) + .build() + .await?; + let (trace, _guard) = capture_planning_spans()?; + let plan = harness.scan().await?; + insta::assert_snapshot!(read_to_string(trace.path())?, @"iceberg.plan_scan{eager=false}: close"); + collect(plan, harness.state().task_ctx()).await?; + insta::assert_snapshot!(read_to_string(trace.path())?, @" + iceberg.plan_scan{eager=false}: close + iceberg.plan_files{eager=false file_tasks=7}: close + "); + Ok(()) +} + +fn capture_planning_spans() -> Result<(NamedTempFile, DefaultGuard)> { + let trace = NamedTempFile::new()?; + let layer = tracing_subscriber::fmt::layer() + .without_time() + .with_target(false) + .with_level(false) + .with_ansi(false) + .with_span_events(FmtSpan::CLOSE) + .with_writer(trace.reopen()?) + .with_filter(filter_fn(|m| m.name().starts_with("iceberg.plan_"))); + let guard = + tracing::subscriber::set_default(tracing_subscriber::registry().with(layer)); + Ok((trace, guard)) +} From 0a42166bf154ea04a9e97c14786bcdb74b24eadf Mon Sep 17 00:00:00 2001 From: alexanderbianchi <75697973+alexanderbianchi@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:01:15 +0000 Subject: [PATCH 2/2] Specify the file discovery stream error type --- crates/datafusion/src/read/file_scan_task_provider/lazy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index c8d1483..45e7cf8 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -241,7 +241,7 @@ fn discover_files(scan: Arc, eager: bool) -> FileScanTaskStream { stream::once(async move { let mut tasks = scan.plan_files().instrument(span.clone()).await?; let mut file_tasks = 0usize; - Ok(stream::poll_fn(move |cx| { + Ok::<_, iceberg::Error>(stream::poll_fn(move |cx| { span.in_scope(|| { let next = tasks.as_mut().poll_next(cx); match &next {