diff --git a/AGENTS.md b/AGENTS.md index 1859768..a77b874 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,7 +90,7 @@ The playground config holds `[[catalogs]]` entries with `name`, `type` (`rest` o ### Read path and filter pushdown -`IcebergDataSource` converts the projection to column names and the filters to a single Iceberg `Predicate` (`read/predicate.rs`). It applies `limit` in-stream, has one output partition, and delegates the actual read to iceberg's `TableScan::to_arrow()`. +`FileScanTaskProviderBuilder` converts the projection to column names and the filters to a single Iceberg `Predicate` (`read/predicate.rs`). Lazy discovery advertises the session's target partition count and shares one discovery stream per execution. In a parallel lazy scan, a producer sends planned file tasks to one unbounded Flume queue. Each partition takes a receiver on first poll; a late reader can claim remaining tasks without restarting discovery. A single-partition scan reads directly without a queue. Use DataFusion's `reset_plan_states` to run the same parallel scan again; partition-open order is not an execution boundary. `IcebergDataSource` reads tasks with a shared `ArrowReader` and applies fetch per partition. Eager planning still collects tasks into round-robin groups. Both providers report every filter as `Inexact`, so DataFusion re-applies the original filters after the scan. The pushed-down predicate therefore only has to be *implied by* the original filter. It may match extra rows but must never drop a row that matches. Unconvertible parts are dropped: an `AND` keeps whichever side converted, while an `OR` needs both sides. Preserve this soundness rule when extending `predicate.rs`; for example, it is why casts and some NaN arithmetic are deliberately not pushed down. Removing a cast is left to DataFusion's simplifier (`unwrap_cast_in_comparison`), which rewrites a comparison only when the result doesn't change. diff --git a/Cargo.lock b/Cargo.lock index b111a48..86a2060 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,6 +87,15 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -102,6 +111,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -1141,6 +1156,12 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.4.3" @@ -1200,6 +1221,33 @@ dependencies = [ "phf", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -1413,6 +1461,41 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -1422,6 +1505,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -2657,6 +2750,9 @@ name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +dependencies = [ + "getrandom 0.4.3", +] [[package]] name = "find-msvc-tools" @@ -2691,6 +2787,18 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "fastrand", + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3241,8 +3349,10 @@ version = "0.12.0" dependencies = [ "async-trait", "bytes", + "criterion", "dashmap", "datafusion", + "flume", "futures", "iceberg", "iceberg-datafusion-test-tools", @@ -4028,6 +4138,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -4091,6 +4207,16 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking" version = "2.2.1" @@ -4229,6 +4355,34 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polyval" version = "0.6.2" @@ -4510,6 +4664,26 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "recursive" version = "0.1.1" @@ -5101,6 +5275,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + [[package]] name = "sqllogictest" version = "0.29.1" @@ -5376,6 +5559,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.12.0" @@ -5895,6 +6088,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -5904,6 +6113,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml index e7fb281..7c166d4 100644 --- a/crates/datafusion/Cargo.toml +++ b/crates/datafusion/Cargo.toml @@ -33,6 +33,7 @@ autotests = false async-trait = { workspace = true } bytes = "1" dashmap = "6" +flume = "0.12" datafusion = { workspace = true } futures = "0.3" iceberg = { workspace = true } @@ -40,7 +41,12 @@ parquet = "59.2" tokio = { workspace = true } uuid = { version = "1.18", features = ["v7"] } +[[bench]] +name = "lazy_scan" +harness = false + [dev-dependencies] +criterion = "0.8" iceberg-datafusion-test-tools = { workspace = true } insta = "1" parquet = "59.2" diff --git a/crates/datafusion/benches/lazy_scan.rs b/crates/datafusion/benches/lazy_scan.rs new file mode 100644 index 0000000..571860d --- /dev/null +++ b/crates/datafusion/benches/lazy_scan.rs @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Time a local Iceberg scan from execution to result, including lazy file discovery. +//! This is a warm, repeated scan of the same table, not a cold object-store benchmark. +//! Both queries project the same data column; only the stopping condition differs. +//! Run this same target on each revision, with a shared `CARGO_TARGET_DIR`: +//! +//! ```sh +//! cargo bench -p iceberg-datafusion --bench lazy_scan -- --save-baseline before +//! cargo bench -p iceberg-datafusion --bench lazy_scan -- --baseline before +//! ``` + +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use datafusion::arrow::array::RecordBatch; +use datafusion::error::Result; +use datafusion::physical_plan::collect; +use iceberg::TableIdent; +use iceberg_datafusion::IcebergTableProviderFactory; +use iceberg_datafusion_test_tools::{ + FixtureCatalog, FixtureTable, IcebergTestHarness, TAXI_TRIPS_PER_DAY, taxi_schema, + taxi_trips, +}; +use tokio::runtime::Builder; + +const DAYS: usize = 160; +const ROWS_PER_FILE: usize = 1_000; + +/// One append per day: 160 manifests, each listing 25 small Parquet files. +async fn many_file_taxi_table() -> Result { + let catalog = FixtureCatalog::try_new().await?; + let name = TableIdent::from_strs(["default", "taxi"]).unwrap(); + let mut table = catalog.table(name, taxi_schema()).create().await?; + for day in 0..DAYS { + let trips = taxi_trips(day)?; + let mut append = table.append(); + for offset in (0..trips.num_rows()).step_by(ROWS_PER_FILE) { + let len = ROWS_PER_FILE.min(trips.num_rows() - offset); + append = append.file(trips.slice(offset, len)); + } + append.commit().await?; + } + Ok(table) +} + +fn lazy_scan(c: &mut Criterion) { + let rt = Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + let harness = rt.block_on(async { + IcebergTestHarness::builder() + .with_table(many_file_taxi_table().await.unwrap()) + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_config_option("datafusion.execution.target_partitions", 4) + .build() + .await + .unwrap() + }); + let context = harness.state().task_ctx(); + let mut group = c.benchmark_group("lazy_scan"); + for (name, sql, rows) in [ + ( + "all_files", + "SELECT fare_amount FROM taxi", + DAYS * TAXI_TRIPS_PER_DAY, + ), + ("limit_one", "SELECT fare_amount FROM taxi LIMIT 1", 1), + ] { + let prepare = || rt.block_on(harness.physical_plan(sql)).unwrap(); + let execute = |plan| rt.block_on(collect(plan, context.clone())).unwrap(); + let output = execute(prepare()); + assert_eq!( + output.iter().map(RecordBatch::num_rows).sum::(), + rows + ); + drop(output); + + group.bench_function(name, |b| { + b.iter_batched(prepare, execute, BatchSize::PerIteration) + }); + } + group.finish(); +} + +criterion_group!(benches, lazy_scan); +criterion_main!(benches); diff --git a/crates/datafusion/src/error.rs b/crates/datafusion/src/error.rs index 9733521..a74a928 100644 --- a/crates/datafusion/src/error.rs +++ b/crates/datafusion/src/error.rs @@ -20,12 +20,17 @@ use iceberg::{Error, ErrorKind}; /// Converts a datafusion error into an iceberg error. pub fn from_datafusion_error(error: DataFusionError) -> Error { - let fallback_message = error.to_string(); - let DataFusionError::Context(context, error) = error else { + // Sibling readers can receive the same planning error. + let mut inner = &error; + while let DataFusionError::Shared(shared) = inner { + inner = shared; + } + let fallback_message = inner.to_string(); + let DataFusionError::Context(context, error) = inner else { return unexpected_datafusion_error(fallback_message); }; - let Some(kind) = parse_iceberg_error_kind(&context) else { + let Some(kind) = parse_iceberg_error_kind(context) else { return unexpected_datafusion_error(fallback_message); }; let DataFusionError::Execution(message) = error.as_ref() else { @@ -95,6 +100,13 @@ mod tests { assert_eq!(roundtripped.to_string(), "DataInvalid => invalid manifest"); } + #[test] + fn preserves_the_kind_of_a_shared_planning_error() { + let error = Error::new(ErrorKind::DataInvalid, "invalid manifest"); + let shared = DataFusionError::Shared(to_datafusion_error(error).into()); + assert_eq!(from_datafusion_error(shared).kind(), ErrorKind::DataInvalid); + } + #[test] fn encodes_iceberg_errors_with_native_datafusion_variants() { let error = diff --git a/crates/datafusion/src/read/file_scan_task_provider/eager.rs b/crates/datafusion/src/read/file_scan_task_provider/eager.rs index bbdd672..f80de8a 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/eager.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/eager.rs @@ -20,6 +20,7 @@ use datafusion::common::stats::Precision; use datafusion::common::{Result, Statistics, exec_err}; use datafusion::physical_plan::Partitioning; use futures::StreamExt; +use futures::stream::BoxStream; use iceberg::scan::FileScanTask; use std::collections::BTreeSet; use std::fmt::Formatter; @@ -70,7 +71,7 @@ impl FileScanTaskProvider for EagerFileScanTaskProvider { self: Arc, partition: usize, _context: FileScanTaskProviderContext, - ) -> Result { + ) -> Result>> { let Some(count) = self.groups().get(partition).map(Vec::len) else { return exec_err!("Invalid Iceberg scan partition {partition}"); }; diff --git a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs index 6843f56..955c5db 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/lazy.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/lazy.rs @@ -14,18 +14,23 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +use std::any::Any; use std::fmt::Formatter; use std::sync::Arc; use datafusion::arrow::datatypes::{Schema, SchemaRef}; +use datafusion::common::runtime::SpawnedTask; use datafusion::common::stats::Precision; -use datafusion::common::{Result, Statistics, exec_err}; +use datafusion::common::{ + DataFusionError, Result, Statistics, exec_datafusion_err, exec_err, +}; use datafusion::execution::TaskContext; use datafusion::physical_plan::Partitioning; use datafusion::prelude::Expr; -use futures::TryStreamExt; +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt, stream}; use iceberg::expr::Predicate; -use iceberg::scan::TableScan; +use iceberg::scan::{FileScanTask, FileScanTaskStream, TableScan}; use iceberg::spec::SnapshotRef; use iceberg::table::Table; @@ -38,8 +43,10 @@ use crate::to_datafusion_error; /// Snapshot summary field from the spec's "Optional Snapshot Summary Fields". const TOTAL_RECORDS: &str = "total-records"; -/// Discovers files while executing, in a single partition: the files of a -/// snapshot that may match a predicate. +/// Plans file tasks during execution. A single partition reads the planning +/// stream directly. Parallel scans use DataFusion sibling state: each reader +/// takes unclaimed files from the same Flume queue, even if it opens late. +/// Use `reset_plan_states` before executing the same parallel plan again. #[derive(Debug, Clone)] pub struct LazyFileScanTaskProvider { table: Table, @@ -48,10 +55,54 @@ pub struct LazyFileScanTaskProvider { snapshot_id: Option, projection: Vec, predicate: Option, + target_partitions: usize, +} + +/// Sibling state for one execution, initialized when its first reader is polled. +enum StreamState { + NotStarted, + Errored(Arc), + Ready(PartitionedFanout), +} + +struct PartitionedFanout { + producer: Arc>, + feeds: Vec>>, +} + +impl PartitionedFanout { + fn new(partitions: usize, mut tasks: FileScanTaskStream) -> Self { + let (tx, rx) = flume::unbounded(); + let producer = SpawnedTask::spawn(async move { + while let Some(task) = tasks.next().await { + if tx.send(task.map_err(to_datafusion_error)).is_err() { + break; + } + } + }); + Self { + producer: Arc::new(producer), + feeds: vec![rx; partitions], + } + } + + fn open(&mut self) -> Result>> { + let feed = self.feeds.pop().ok_or_else(|| { + exec_datafusion_err!("All file task streams have already been opened") + })?; + // Keep the producer alive even if the plan drops before its readers. + let producer = Arc::clone(&self.producer); + Ok(feed + .into_stream() + .inspect(move |_| { + let _ = &producer; + }) + .boxed()) + } } impl LazyFileScanTaskProvider { - /// Consumes discovery and retains its tasks in round-robin groups, using the + /// Collects the planned tasks into round-robin groups, using the /// context's target partition count. The result no longer retains the table. /// Empty discovery produces one empty group. Dropping this future drops the /// discovery stream; this method does not spawn a background collector. @@ -65,12 +116,7 @@ impl LazyFileScanTaskProvider { "Iceberg eager planning requires at least one target partition" ); } - let mut tasks = Arc::new(self).open( - 0, - FileScanTaskProviderContext { - task_context: &context, - }, - )?; + let mut tasks = self.scan.plan_files().await.map_err(to_datafusion_error)?; let mut groups = Vec::new(); let mut partition = 0; while let Some(task) = tasks.try_next().await.map_err(to_datafusion_error)? { @@ -200,6 +246,12 @@ impl FileScanTaskProviderBuilder { .iter() .map(|field| field.name().clone()) .collect(); + let target_partitions = self.context.session_config().target_partitions(); + if target_partitions == 0 { + return exec_err!( + "Iceberg file planning requires at least one target partition" + ); + } // TableScan resolves and retains the snapshot without discovering files. // Keep that same scan for execution and its snapshot for statistics. let mut scan = self.table.scan().select(&projection); @@ -217,6 +269,7 @@ impl FileScanTaskProviderBuilder { projection, snapshot_id: self.snapshot_id, predicate: self.predicate, + target_partitions, }; if IcebergDataFusionConfig::from_config_or_default(self.context.session_config()) .execution @@ -231,22 +284,63 @@ impl FileScanTaskProviderBuilder { impl FileScanTaskProvider for LazyFileScanTaskProvider { fn partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) + Partitioning::UnknownPartitioning(self.target_partitions) + } + + fn create_sibling_state(&self) -> Option> { + if self.target_partitions == 1 { + return None; + } + Some(Arc::new(tokio::sync::Mutex::new(StreamState::NotStarted))) } fn open( self: Arc, partition: usize, - _ctx: FileScanTaskProviderContext, - ) -> Result { - if partition != 0 { + context: FileScanTaskProviderContext, + ) -> Result>> { + if partition >= self.target_partitions { return exec_err!("Invalid Iceberg scan partition {partition}"); } - let scan = Arc::clone(&self.scan); - // Files are discovered when the stream is first polled, not here. - Ok(Box::pin( - futures::stream::once(async move { scan.plan_files().await }).try_flatten(), - )) + if self.target_partitions == 1 { + return Ok(stream::once(async move { self.scan.plan_files().await }) + .try_flatten() + .map_err(to_datafusion_error) + .boxed()); + } + let state = context + .sibling_state + .and_then(|state| state.downcast::>().ok()) + .ok_or_else(|| { + exec_datafusion_err!( + "Parallel lazy scans require sibling state from DataSourceExec" + ) + })?; + let feed = async move { + let mut state = state.lock().await; + match &mut *state { + StreamState::Ready(fanout) => return fanout.open(), + StreamState::Errored(error) => { + return Err(DataFusionError::Shared(Arc::clone(error))); + } + StreamState::NotStarted => {} + } + match self.scan.plan_files().await { + Ok(tasks) => { + let mut fanout = + PartitionedFanout::new(self.target_partitions, tasks); + let feed = fanout.open()?; + *state = StreamState::Ready(fanout); + Ok(feed) + } + Err(error) => { + let error = Arc::new(to_datafusion_error(error)); + *state = StreamState::Errored(Arc::clone(&error)); + Err(DataFusionError::Shared(error)) + } + } + }; + Ok(stream::once(feed).try_flatten().boxed()) } /// Estimates the rows the scan returns before its predicate and fetch, @@ -266,13 +360,17 @@ impl FileScanTaskProvider for LazyFileScanTaskProvider { partition: Option, schema: &Schema, ) -> Result { - if partition.is_some_and(|partition| partition != 0) { + if partition.is_some_and(|partition| partition >= self.target_partitions) { return exec_err!("Invalid Iceberg scan partition {partition:?}"); } let statistics = Statistics::new_unknown(schema); let Some(snapshot) = self.snapshot() else { return Ok(statistics.with_num_rows(Precision::Exact(0))); }; + // Before discovery, no individual reader has an assigned share of the files. + if partition.is_some() && self.target_partitions > 1 { + return Ok(statistics); + } let num_rows = match snapshot .summary() .additional_properties diff --git a/crates/datafusion/src/read/file_scan_task_provider/provider.rs b/crates/datafusion/src/read/file_scan_task_provider/provider.rs index 1f8ba15..452ba43 100644 --- a/crates/datafusion/src/read/file_scan_task_provider/provider.rs +++ b/crates/datafusion/src/read/file_scan_task_provider/provider.rs @@ -22,6 +22,8 @@ use datafusion::arrow::datatypes::Schema; use datafusion::common::{Result, Statistics}; use datafusion::execution::TaskContext; use datafusion::physical_plan::Partitioning; +use futures::stream::BoxStream; +use iceberg::scan::FileScanTask; /// Supplies the file tasks an [`IcebergDataSource`](crate::read::IcebergDataSource) /// reads, split into output partitions. @@ -45,13 +47,21 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync { /// projected schema. fn partitioning(&self) -> Partitioning; + /// Creates state shared by the partitions of one execution, passed to + /// [`Self::open`] through [`FileScanTaskProviderContext::sibling_state`]. + /// The data source creates it once per execution, so re-executing a plan + /// after resetting its state starts afresh. Returns `None` by default. + fn create_sibling_state(&self) -> Option> { + None + } + /// Streams the file tasks of `partition`, which is below the /// [`Self::partitioning`] partition count. fn open( self: Arc, partition: usize, context: FileScanTaskProviderContext, - ) -> Result; + ) -> Result>>; /// Statistics before the source's fetch, without discovering or reading files. /// `None` requests the whole assignment; `Some(i)` requests partition `i`, @@ -85,4 +95,7 @@ pub trait FileScanTaskProvider: Any + Debug + Send + Sync { /// table, belong to the provider rather than the data source. pub struct FileScanTaskProviderContext<'a> { pub task_context: &'a Arc, + /// The state from [`FileScanTaskProvider::create_sibling_state`] for this + /// execution, or `None` when the caller opens partitions on their own. + pub sibling_state: Option>, } diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index 30bb423..cb9dd07 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -15,17 +15,19 @@ // specific language governing permissions and limitations // under the License. +use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; use super::file_scan_task_provider::{FileScanTaskProvider, FileScanTaskProviderContext}; -use crate::to_datafusion_error; +use crate::{from_datafusion_error, to_datafusion_error}; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::stats::Precision; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{Statistics, exec_err}; -use datafusion::datasource::source::DataSource; +use datafusion::config::ConfigOptions; +use datafusion::datasource::source::{DataSource, OpenArgs}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::projection::ProjectionExprs; @@ -35,7 +37,7 @@ use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSe use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayFormatType, Partitioning}; use datafusion::prelude::SessionConfig; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use iceberg::arrow::{ArrowReader, ArrowReaderBuilder}; use iceberg::io::FileIO; @@ -231,6 +233,22 @@ impl DataSource for IcebergDataSource { partition: usize, context: Arc, ) -> Result { + self.open_with_args(OpenArgs::new(partition, context)) + } + + fn create_sibling_state( + &self, + _config: &ConfigOptions, + ) -> Option> { + self.file_scan_task_provider.create_sibling_state() + } + + fn open_with_args(&self, args: OpenArgs) -> Result { + let OpenArgs { + partition, + context, + sibling_state, + } = args; if partition >= self.output_partitioning().partition_count() { return exec_err!("Invalid Iceberg scan partition {partition}"); } @@ -238,12 +256,13 @@ impl DataSource for IcebergDataSource { partition, FileScanTaskProviderContext { task_context: &context, + sibling_state, }, )?; let stream = self .reader .clone() - .read(tasks) + .read(tasks.map_err(from_datafusion_error).boxed()) .map_err(to_datafusion_error)? .stream() .map_err(to_datafusion_error); @@ -344,7 +363,7 @@ mod tests { use datafusion::common::Statistics; use datafusion::common::stats::Precision; use datafusion::datasource::source::DataSource; - use datafusion::prelude::SessionContext; + use datafusion::prelude::{SessionConfig, SessionContext}; use iceberg::io::FileIO; use iceberg::spec::{ FormatVersion, MAIN_BRANCH, NestedField, Operation, PartitionSpec, PrimitiveType, @@ -421,7 +440,9 @@ mod tests { DataType::Int64, false, )])); - let context = SessionContext::new(); + let context = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(1), + ); let provider = FileScanTaskProviderBuilder::new( context.task_ctx(), table.clone(), diff --git a/crates/datafusion/tests/catalog_provider.rs b/crates/datafusion/tests/catalog_provider.rs index c80934d..3462d1c 100644 --- a/crates/datafusion/tests/catalog_provider.rs +++ b/crates/datafusion/tests/catalog_provider.rs @@ -78,7 +78,10 @@ async fn test_provider_plan_stream_schema() -> Result<(), Box> { let task_ctx = Arc::new(df.task_ctx()); let plan = df.create_physical_plan().await.unwrap(); - assert_eq!(plan.properties().partitioning.partition_count(), 1); + assert_eq!( + plan.properties().partitioning.partition_count(), + ctx.copied_config().target_partitions() + ); let stream = plan.execute(0, task_ctx).unwrap(); // Ensure both the plan and the stream conform to the same schema diff --git a/crates/datafusion/tests/file_scan_task_provider.rs b/crates/datafusion/tests/file_scan_task_provider.rs index c9c3447..a1d0a91 100644 --- a/crates/datafusion/tests/file_scan_task_provider.rs +++ b/crates/datafusion/tests/file_scan_task_provider.rs @@ -22,14 +22,18 @@ use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch}; use datafusion::common::Result; use datafusion::datasource::source::{DataSource, DataSourceExec}; use datafusion::execution::TaskContext; -use datafusion::physical_plan::{ExecutionPlan, displayable}; +use datafusion::physical_plan::execution_plan::reset_plan_states; +use datafusion::physical_plan::{ + ExecutionPlan, ExecutionPlanProperties, collect, displayable, +}; use futures::TryStreamExt; use iceberg::TableIdent; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg_datafusion::read::{EagerFileScanTaskProvider, IcebergDataSource}; use iceberg_datafusion::{IcebergDataFusionConfig, IcebergTableProviderFactory}; use iceberg_datafusion_test_tools::{ - FixtureCatalog, IcebergTestHarness, create_taxi_table, + FixtureCatalog, FixtureTable, IcebergTestHarness, ReadCountingStorageFactory, + create_taxi_table, }; #[tokio::test] @@ -57,8 +61,7 @@ async fn lazy_scan_filters_and_projects_taxi_rows() -> Result<()> { RepartitionExec: partitioning=Hash([pickup_date@0], 2), input_partitions=2 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 = 2024-01-10 OR pickup_date@0 = 2024-01-11 - RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) "); insta::assert_snapshot!(rows, @" +-------------+-------+ @@ -192,17 +195,7 @@ async fn replacing_the_provider_preserves_the_fetch_limit() -> Result<()> { #[tokio::test] async fn eager_file_statistics_apply_fetch_per_partition() -> Result<()> { - let fixtures = FixtureCatalog::try_new().await?; - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), - ]) - .build() - .unwrap(); - let mut table = fixtures - .table(TableIdent::from_strs(["default", "items"]).unwrap(), schema) - .create() - .await?; + let mut table = empty_items().await?; table .append() .file(id_batch(vec![1, 2, 3])?) @@ -251,6 +244,7 @@ async fn the_scan_keeps_its_construction_time_reader_settings() -> Result<()> { let harness = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) + .with_config_option("datafusion.execution.target_partitions", 1) .with_config_option("datafusion.execution.batch_size", 2000) .build() .await?; @@ -269,6 +263,131 @@ async fn the_scan_keeps_its_construction_time_reader_settings() -> Result<()> { Ok(()) } +#[tokio::test] +async fn reset_scans_read_the_same_files_independently() -> Result<()> { + let mut table = empty_items().await?; + table + .append() + .file(id_batch(vec![1, 2])?) + .file(id_batch(vec![3, 4])?) + .commit() + .await?; + let harness = IcebergTestHarness::builder() + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_table(table) + .with_config_option("datafusion.execution.target_partitions", 4) + .build() + .await?; + let scan = harness.scan().await?; + assert_eq!(scan.output_partitioning().partition_count(), 4); + let next_run = reset_plan_states(scan.clone())?; + let context = harness.state().task_ctx(); + let (original_rows, reset_rows) = + tokio::try_join!(collect(scan, context.clone()), collect(next_run, context),)?; + + let expected = [ + "+----+", "| id |", "+----+", "| 1 |", "| 2 |", "| 3 |", "| 4 |", "+----+", + ]; + datafusion::assert_batches_sorted_eq!(expected, &original_rows); + datafusion::assert_batches_sorted_eq!(expected, &reset_rows); + Ok(()) +} + +#[tokio::test] +async fn a_late_partition_reads_only_the_unclaimed_file() -> Result<()> { + let mut table = empty_items().await?; + table + .append() + .file(id_batch(vec![1])?) + .file(id_batch(vec![2])?) + .commit() + .await?; + let harness = IcebergTestHarness::builder() + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_table(table) + .with_config_option("datafusion.execution.target_partitions", 2) + .build() + .await?; + let scan = harness.scan().await?; + let context = harness.state().task_ctx(); + + let mut active = scan.execute(0, context.clone())?; + let first = active.try_next().await?.expect("one row in the first file"); + assert_eq!(first.num_rows(), 1); + drop(active); + + let mut rows = scan.execute(1, context)?.try_collect::>().await?; + rows.push(first); + datafusion::assert_batches_sorted_eq!( + ["+----+", "| id |", "+----+", "| 1 |", "| 2 |", "+----+"], + &rows + ); + Ok(()) +} + +#[tokio::test] +async fn lazy_scan_shares_a_position_delete_file_across_data_files() -> Result<()> { + let mut table = empty_items().await?; + let files = table + .append() + .file(id_batch(vec![1, 2])?) + .file(id_batch(vec![3, 4])?) + .commit() + .await?; + let deletes = table + .delete_positions(&[(&files[0], 0), (&files[1], 0)]) + .await?; + let storage = ReadCountingStorageFactory::default(); + let (_, rows) = IcebergTestHarness::builder() + .with_iceberg_table_factory( + IcebergTableProviderFactory::new_with_storage_factory(Arc::new( + storage.clone(), + )), + ) + .with_table(table) + .with_config_option("datafusion.execution.target_partitions", 2) + .query("SELECT id FROM items ORDER BY id") + .await?; + insta::assert_snapshot!(rows, @" + +----+ + | id | + +----+ + | 2 | + | 4 | + +----+ + "); + assert_eq!(storage.file_opens(deletes.file_path()), 1); + Ok(()) +} + +#[tokio::test] +async fn lazy_scan_without_a_snapshot_is_empty() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_table(empty_items().await?) + .with_config_option("datafusion.execution.target_partitions", 4) + .build() + .await?; + let plan = harness.scan().await?; + assert_eq!(plan.output_partitioning().partition_count(), 4); + assert!(collect(plan, harness.state().task_ctx()).await?.is_empty()); + Ok(()) +} + +async fn empty_items() -> Result { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + FixtureCatalog::try_new() + .await? + .table(TableIdent::from_strs(["default", "items"]).unwrap(), schema) + .create() + .await +} + fn id_batch(values: Vec) -> Result { Ok(RecordBatch::try_from_iter([( "id", diff --git a/crates/datafusion/tests/filter_pushdown.rs b/crates/datafusion/tests/filter_pushdown.rs index 47718c7..8a12409 100644 --- a/crates/datafusion/tests/filter_pushdown.rs +++ b/crates/datafusion/tests/filter_pushdown.rs @@ -49,8 +49,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: pickup_date@2 = 2024-01-10 AND (payment_type@1 = 1 OR payment_type@1 = 2) AND trip_distance@0 >= 2, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, payment_type, pickup_date], provider=lazy, predicate=((pickup_date = 2024-01-10) AND ((payment_type = 1) OR (payment_type = 2))) AND (trip_distance >= 2) + DataSourceExec: format=iceberg, projection=[trip_distance, payment_type, pickup_date], provider=lazy, predicate=((pickup_date = 2024-01-10) AND ((payment_type = 1) OR (payment_type = 2))) AND (trip_distance >= 2) "); insta::assert_snapshot!(batches, @r" +-------+ @@ -86,8 +85,7 @@ mod tests { RepartitionExec: partitioning=Hash([pickup_date@0], 16), input_partitions=16 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 = 2024-01-10 OR pickup_date@0 = 2024-01-11 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) "); insta::assert_snapshot!(batches, @r" +-------------+-------+ @@ -115,8 +113,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: passenger_count@0 IS NULL, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[passenger_count], provider=lazy, predicate=passenger_count IS NULL + DataSourceExec: format=iceberg, projection=[passenger_count], provider=lazy, predicate=passenger_count IS NULL "); insta::assert_snapshot!(batches, @r" +-------+ @@ -148,8 +145,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: pickup_date@1 = 2024-01-10 AND trip_distance@0 + 1 > 3, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------+ @@ -176,8 +172,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] FilterExec: trip_distance@0 + 1 > 20, projection=[] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance], provider=lazy + DataSourceExec: format=iceberg, projection=[trip_distance], provider=lazy "); insta::assert_snapshot!(batches, @r" +-------+ diff --git a/crates/datafusion/tests/limit_pushdown.rs b/crates/datafusion/tests/limit_pushdown.rs index 9a721db..2e1a861 100644 --- a/crates/datafusion/tests/limit_pushdown.rs +++ b/crates/datafusion/tests/limit_pushdown.rs @@ -43,8 +43,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST], fetch=3 SortExec: TopK(fetch=3), expr=[pickup_at@1 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@4 = 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, passenger_count, trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, passenger_count, trip_distance, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+-----------------+---------------+-------------+ @@ -70,8 +69,7 @@ mod tests { insta::assert_snapshot!(plan, @r" CoalescePartitionsExec: fetch=3 FilterExec: pickup_date@0 = 2024-01-10, fetch=3 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------+ @@ -91,22 +89,20 @@ mod tests { let (plan, batches) = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) - .query("SELECT pickup_date FROM taxi LIMIT 3") + .build() + .await? + .query_batches("SELECT pickup_date FROM taxi LIMIT 3") .await?; insta::assert_snapshot!(plan, @r" - CooperativeExec + CoalescePartitionsExec: fetch=3 DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, fetch=3 "); - insta::assert_snapshot!(batches, @r" - +-------------+ - | pickup_date | - +-------------+ - | 2024-01-08 | - | 2024-01-08 | - | 2024-01-08 | - +-------------+ - "); + // No ORDER BY: parallel partitions need not return the first day. + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 3 + ); Ok(()) } @@ -116,7 +112,8 @@ mod tests { let (_, explain) = IcebergTestHarness::builder() .with_iceberg_table_factory(IcebergTableProviderFactory::new()) .with_table(create_taxi_table().await?) - // Only row counts, so the output doesn't vary between runs. + // With one partition, source rows are deterministic even under LIMIT. + .with_config_option("datafusion.execution.target_partitions", 1) .with_config_option("datafusion.explain.analyze_categories", "rows") .query("EXPLAIN ANALYZE SELECT pickup_date FROM taxi LIMIT 3") .await?; @@ -144,6 +141,7 @@ mod tests { )), ) .with_table(create_taxi_table().await?) + .with_config_option("datafusion.execution.target_partitions", 2) .build() .await?; harness.query_batches("SELECT * FROM taxi").await?; @@ -151,8 +149,8 @@ mod tests { harness.query_batches("SELECT * FROM taxi LIMIT 3").await?; let limited_scan_reads = storage.take_reads(); - // The exact counts depend on how many data files iceberg opens at - // once, which follows the runner's CPU count, so compare the two scans. + // Two partitions can open fewer than the seven files before LIMIT + // cancels the scan. Compare totals, not scheduler-dependent exact counts. assert!( limited_scan_reads < full_scan_reads, "a scan with LIMIT 3 read {limited_scan_reads} Parquet ranges, \ @@ -201,8 +199,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@2 ASC NULLS LAST], fetch=5 SortExec: TopK(fetch=5), expr=[pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@2 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@3 = 2024-01-10, projection=[vendor_id@0, pickup_at@1, pickup_location_id@2] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+ diff --git a/crates/datafusion/tests/plan_nodes.rs b/crates/datafusion/tests/plan_nodes.rs index f5d469d..8e5bc6e 100644 --- a/crates/datafusion/tests/plan_nodes.rs +++ b/crates/datafusion/tests/plan_nodes.rs @@ -107,7 +107,7 @@ async fn run_source( ctx: &SessionContext, ) -> Result> { let exec = DataSourceExec::from_data_source(source.clone()); - run(exec.as_ref(), ctx).await + run(&CoalescePartitionsExec::new(exec), ctx).await } /// Returns the first node of type `T` in `plan`, depth first. diff --git a/crates/datafusion/tests/projection_pushdown.rs b/crates/datafusion/tests/projection_pushdown.rs index dc4f996..0e249e9 100644 --- a/crates/datafusion/tests/projection_pushdown.rs +++ b/crates/datafusion/tests/projection_pushdown.rs @@ -51,8 +51,7 @@ mod tests { RepartitionExec: partitioning=Hash([pickup_date@0], 16), input_partitions=16 AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] FilterExec: pickup_date@0 >= 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date >= 2024-01-10 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=pickup_date >= 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------+-------+ @@ -88,8 +87,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@2 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST], fetch=3 SortExec: TopK(fetch=3), expr=[pickup_at@2 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@3 = 2024-01-10, projection=[vendor_id@0, pickup_location_id@2, pickup_at@1] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, pickup_location_id, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+--------------------+ @@ -122,8 +120,7 @@ mod tests { CoalescePartitionsExec AggregateExec: mode=Partial, gby=[], aggr=[max(taxi.trip_distance * taxi.fare_amount)] FilterExec: pickup_date@2 = 2024-01-10, projection=[trip_distance@0, fare_amount@1] - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[trip_distance, fare_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[trip_distance, fare_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-------------------+ @@ -154,8 +151,7 @@ mod tests { SortPreservingMergeExec: [pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@5 ASC NULLS LAST], fetch=1 SortExec: TopK(fetch=1), expr=[pickup_at@1 ASC NULLS LAST, vendor_id@0 ASC NULLS LAST, pickup_location_id@5 ASC NULLS LAST], preserve_partitioning=[true] FilterExec: pickup_date@12 = 2024-01-10 - RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 - DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, dropoff_at, passenger_count, trip_distance, pickup_location_id, dropoff_location_id, payment_type, fare_amount, tip_amount, tolls_amount, total_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 + DataSourceExec: format=iceberg, projection=[vendor_id, pickup_at, dropoff_at, passenger_count, trip_distance, pickup_location_id, dropoff_location_id, payment_type, fare_amount, tip_amount, tolls_amount, total_amount, pickup_date], provider=lazy, predicate=pickup_date = 2024-01-10 "); insta::assert_snapshot!(batches, @r" +-----------+---------------------+---------------------+-----------------+---------------+--------------------+---------------------+--------------+-------------+------------+--------------+--------------+-------------+ diff --git a/crates/datafusion/tests/schema_evolution.rs b/crates/datafusion/tests/schema_evolution.rs index bb865a5..0dfb145 100644 --- a/crates/datafusion/tests/schema_evolution.rs +++ b/crates/datafusion/tests/schema_evolution.rs @@ -24,7 +24,7 @@ use datafusion::arrow::compute::{concat_batches, sort_to_indices, take_record_ba use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::datasource::TableProvider; use datafusion::execution::context::SessionContext; -use datafusion::physical_plan::common::collect; +use datafusion::physical_plan::collect; use iceberg::TableIdent; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::transaction::{AddColumn, ApplyTransactionAction, Transaction}; @@ -100,8 +100,11 @@ async fn test_scan_after_schema_evolution_reads_provider_columns() // The rows come from two data files, which may be read in either order. // They are joined under their own schema, not the plan's, so that a column // the plan does not report would show. - assert_eq!(plan.properties().partitioning.partition_count(), 1); - let batches = collect(plan.execute(0, ctx.task_ctx())?).await?; + assert_eq!( + plan.properties().partitioning.partition_count(), + ctx.copied_config().target_partitions() + ); + let batches = collect(plan, ctx.task_ctx()).await?; let batch = concat_batches(&batches[0].schema(), &batches)?; let order = sort_to_indices(batch.column_by_name("foo1").unwrap(), None, None)?; let sorted = take_record_batch(&batch, &order)?; diff --git a/crates/datafusion/tests/statistics.rs b/crates/datafusion/tests/statistics.rs index eafa31b..c41af96 100644 --- a/crates/datafusion/tests/statistics.rs +++ b/crates/datafusion/tests/statistics.rs @@ -21,6 +21,23 @@ mod tests { use iceberg_datafusion::{IcebergDataFusionConfig, IcebergTableProviderFactory}; use iceberg_datafusion_test_tools::{IcebergTestHarness, create_taxi_table}; + #[tokio::test] + async fn lazy_snapshot_count_belongs_to_the_scan_not_either_reader() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_iceberg_table_factory(IcebergTableProviderFactory::new()) + .with_table(create_taxi_table().await?) + .with_config_option("datafusion.execution.target_partitions", 2) + .build() + .await?; + let plan = harness.physical_plan("SELECT vendor_id FROM taxi").await?; + insta::assert_snapshot!(harness.source_statistics(&plan)?, @" + all partitions (2): Rows=Inexact(175000), Bytes=Inexact(700000), [(Col[0]:)] + partition 0: Rows=Absent, Bytes=Absent, [(Col[0]:)] + partition 1: Rows=Absent, Bytes=Absent, [(Col[0]:)] + "); + Ok(()) + } + #[tokio::test] async fn reports_the_snapshot_row_count_to_datafusion() -> Result<()> { let (_, explain) = IcebergTestHarness::builder() diff --git a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt b/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt index bac4267..9636326 100644 --- a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt +++ b/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt @@ -51,7 +51,7 @@ logical_plan 01)Limit: skip=0, fetch=3 02)--TableScan: default.default.query_test_table projection=[id, name, score, category], fetch=3 physical_plan -01)CooperativeExec +01)CoalescePartitionsExec: fetch=3 02)--DataSourceExec: format=iceberg, projection=[id, name, score, category], provider=lazy, fetch=3 # Test SELECT * with ORDER BY and LIMIT diff --git a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt index 6a22dd9..205be33 100644 --- a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt @@ -38,7 +38,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = true # Query with is_active = true @@ -58,7 +58,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = false # Query with is_active = false @@ -77,7 +77,7 @@ logical_plan 02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] physical_plan 01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, is_active, description], provider=lazy, predicate=is_active = false # Query with is_active != true (includes false and NULL) diff --git a/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt index d25e445..611172f 100644 --- a/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/cast_predicate_pushdown.slt @@ -40,7 +40,7 @@ logical_plan 03)----TableScan: default.default.test_cast_table projection=[id, d], partial_filters=[CAST(default.default.test_cast_table.d AS Timestamp(ns)) < TimestampNanosecond(1672920000000000000, None)] physical_plan 01)FilterExec: CAST(d@1 AS Timestamp(ns)) < 1672920000000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, d], provider=lazy query I rowsort @@ -73,7 +73,7 @@ logical_plan 03)----TableScan: default.default.test_cast_table projection=[id, n], partial_filters=[CAST(default.default.test_cast_table.n AS Utf8) < Utf8("5")] physical_plan 01)FilterExec: CAST(n@1 AS Utf8) < 5, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, n], provider=lazy query I rowsort @@ -108,7 +108,7 @@ logical_plan 03)----TableScan: default.default.test_float_cast_table projection=[id, x], partial_filters=[CAST(default.default.test_float_cast_table.x AS Float64) > Float64(0.1)] physical_plan 01)FilterExec: CAST(x@1 AS Float64) > 0.1, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, x], provider=lazy query I rowsort @@ -137,7 +137,7 @@ logical_plan 03)----TableScan: default.default.test_isnan_cast_table projection=[id, s], partial_filters=[NOT isnan(CAST(default.default.test_isnan_cast_table.s AS Float64))] physical_plan 01)FilterExec: NOT isnan(CAST(s@1 AS Float64)), projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, s], provider=lazy query I rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt index dc4c36a..d3c93f6 100644 --- a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt @@ -36,7 +36,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name LIKE Utf8("Al%")] physical_plan 01)FilterExec: name@1 LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name STARTS WITH "Al" # Test LIKE filtering with case-sensitive match @@ -55,7 +55,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name NOT LIKE Utf8("Al%")] physical_plan 01)FilterExec: name@1 NOT LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name NOT STARTS WITH "Al" # Test NOT LIKE filtering @@ -129,7 +129,7 @@ logical_plan 02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name LIKE Utf8("Al\%%")] physical_plan 01)FilterExec: name@1 LIKE Al\%% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, name], provider=lazy, predicate=name STARTS WITH "Al%" query IT rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt index a23ad57..166368a 100644 --- a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt @@ -49,7 +49,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts = TimestampNanosecond(1672921800000000000, None)] physical_plan 01)FilterExec: ts@1 = 1672921800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts = 2023-01-05 12:30:00 # Verify timestamp equality filtering works @@ -67,7 +67,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts > TimestampNanosecond(1673308800000000000, None)] physical_plan 01)FilterExec: ts@1 > 1673308800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts > 2023-01-10 00:00:00 # Verify timestamp greater than filtering @@ -96,7 +96,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts >= TimestampNanosecond(1672876800000000000, None), default.default.test_timestamp_table.ts <= TimestampNanosecond(1673827199000000000, None)] physical_plan 01)FilterExec: ts@1 >= 1672876800000000000 AND ts@1 <= 1673827199000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59) # Test timestamp range predicate filtering @@ -161,7 +161,7 @@ logical_plan 02)--TableScan: default.default.test_timestamp_micros projection=[id, ts], partial_filters=[default.default.test_timestamp_micros.ts > TimestampMicrosecond(1672531200000000, None)] physical_plan 01)FilterExec: ts@1 > 1672531200000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts], provider=lazy, predicate=ts > 2023-01-01 00:00:00 query I? @@ -206,7 +206,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_us], partial_filters=[default.default.test_timestamptz.ts_us > TimestampMicrosecond(1672876800000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_us@1 > 1672876800000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_us], provider=lazy, predicate=ts_us > 2023-01-05 00:00:00 UTC query I @@ -223,7 +223,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_ns], partial_filters=[default.default.test_timestamptz.ts_ns > TimestampNanosecond(1672876800000000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_ns@1 > 1672876800000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_ns], provider=lazy, predicate=ts_ns > 2023-01-05 00:00:00 UTC query I @@ -240,7 +240,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_us], partial_filters=[default.default.test_timestamptz.ts_us > TimestampMicrosecond(1672876800000000, Some("+00:00"))] physical_plan 01)FilterExec: ts_us@1 > 1672876800000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_us], provider=lazy, predicate=ts_us > 2023-01-05 00:00:00 UTC query I @@ -257,7 +257,7 @@ logical_plan 03)----TableScan: default.default.test_timestamptz projection=[id, ts_naive], partial_filters=[default.default.test_timestamptz.ts_naive > TimestampNanosecond(1672876800000000000, None)] physical_plan 01)FilterExec: ts_naive@1 > 1672876800000000000, projection=[id@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, ts_naive], provider=lazy, predicate=ts_naive > 2023-01-05 00:00:00 query I diff --git a/crates/test-tools/src/append.rs b/crates/test-tools/src/append.rs index bb33b5f..89b6eae 100644 --- a/crates/test-tools/src/append.rs +++ b/crates/test-tools/src/append.rs @@ -19,7 +19,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use datafusion::arrow::array::{RecordBatch, new_null_array}; +use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray, new_null_array}; use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::{exec_err, not_impl_err}; @@ -29,9 +29,12 @@ use iceberg::arrow::{RecordBatchPartitionSplitter, schema_to_arrow_schema}; use iceberg::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestListWriter, ManifestWriterBuilder, Operation, Snapshot, SnapshotSummaryCollector, Summary, - UNASSIGNED_SEQUENCE_NUMBER, + TableMetadata, UNASSIGNED_SEQUENCE_NUMBER, }; use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; +use iceberg::writer::base_writer::position_delete_writer::{ + PositionDeleteFileWriterBuilder, position_delete_schema, +}; use iceberg::writer::file_writer::ParquetWriterBuilder; use iceberg::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, @@ -94,14 +97,7 @@ impl<'a> Append<'a> { let schema = metadata.current_schema().clone(); let partition_spec = metadata.default_partition_spec().clone(); let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).map_err(df_err)?); - let parent_snapshot_id = metadata.current_snapshot_id(); - let snapshot_id = metadata - .snapshots() - .map(|s| s.snapshot_id()) - .max() - .unwrap_or(0) - + 1; - let sequence_number = metadata.next_sequence_number(); + let snapshot_id = next_snapshot_id(metadata); let splitter = if partition_spec.is_unpartitioned() { None @@ -151,7 +147,89 @@ impl<'a> Append<'a> { data_files.extend(writer.close().await.map_err(df_err)?); } - let mut manifest_writer = ManifestWriterBuilder::new( + self.table + .commit_files(&data_files, Operation::Append) + .await?; + Ok(data_files) + } +} + +impl FixtureTable { + /// Commits one shared position-delete file for an unpartitioned V2 table. + /// Each pair identifies a zero-based row in a previously appended file. + pub async fn delete_positions( + &mut self, + rows: &[(&DataFile, i64)], + ) -> Result { + let table = self.table(); + let metadata = table.metadata(); + if metadata.format_version() != FormatVersion::V2 + || !metadata.default_partition_spec().is_unpartitioned() + { + return not_impl_err!( + "fixture position deletes require an unpartitioned V2 table" + ); + } + if rows.is_empty() + || rows + .iter() + .any(|(file, pos)| *pos < 0 || *pos as u64 >= file.record_count()) + { + return exec_err!("expected at least one valid position to delete"); + } + let mut rows: Vec<_> = rows + .iter() + .map(|(file, pos)| (file.file_path(), *pos)) + .collect(); + rows.sort_unstable(); + let (paths, positions): (Vec<_>, Vec<_>) = rows.into_iter().unzip(); + let schema = position_delete_schema(); + let batch = RecordBatch::try_new( + Arc::new(schema_to_arrow_schema(&schema).map_err(df_err)?), + vec![ + Arc::new(StringArray::from(paths)), + Arc::new(Int64Array::from(positions)), + ], + )?; + let mut writer = + PositionDeleteFileWriterBuilder::new(RollingFileWriterBuilder::new( + ParquetWriterBuilder::new(WriterProperties::default(), schema), + usize::MAX, + table.file_io().clone(), + DefaultLocationGenerator::new(metadata).map_err(df_err)?, + DefaultFileNameGenerator::new( + format!("deletes-{}", next_snapshot_id(metadata)), + None, + DataFileFormat::Parquet, + ), + )) + .build(None) + .await + .map_err(df_err)?; + writer.write(batch).await.map_err(df_err)?; + let files = writer.close().await.map_err(df_err)?; + let Ok([delete]) = <[DataFile; 1]>::try_from(files) else { + return exec_err!("expected one position-delete file"); + }; + self.commit_files(std::slice::from_ref(&delete), Operation::Delete) + .await?; + Ok(delete) + } + + async fn commit_files( + &mut self, + files: &[DataFile], + operation: Operation, + ) -> Result<()> { + let table = self.table(); + let metadata = table.metadata(); + let file_io = table.file_io(); + let schema = metadata.current_schema().clone(); + let partition_spec = metadata.default_partition_spec().clone(); + let snapshot_id = next_snapshot_id(metadata); + let parent_snapshot_id = metadata.current_snapshot_id(); + let sequence_number = metadata.next_sequence_number(); + let builder = ManifestWriterBuilder::new( file_io .new_output(format!( "{}/metadata/snap-{snapshot_id}-m0.avro", @@ -161,10 +239,14 @@ impl<'a> Append<'a> { Some(snapshot_id), schema.clone(), partition_spec.as_ref().clone(), - ) - .build_v2_data(); + ); + let mut manifest_writer = if operation == Operation::Delete { + builder.build_v2_deletes() + } else { + builder.build_v2_data() + }; let mut summary = SnapshotSummaryCollector::default(); - for data_file in &data_files { + for data_file in files { manifest_writer .add_file(data_file.clone(), UNASSIGNED_SEQUENCE_NUMBER) .map_err(df_err)?; @@ -210,7 +292,7 @@ impl<'a> Append<'a> { .with_timestamp_ms(now_ms()) .with_manifest_list(manifest_list) .with_summary(Summary { - operation: Operation::Append, + operation, additional_properties: with_totals( summary.build(), metadata.current_snapshot().map(|parent| parent.summary()), @@ -220,17 +302,25 @@ impl<'a> Append<'a> { .build(); let metadata = metadata .clone() - .into_builder(Some(self.table.metadata_location()?.to_string())) + .into_builder(Some(self.metadata_location()?.to_string())) .set_branch_snapshot(snapshot, MAIN_BRANCH) .map_err(df_err)? .build() .map_err(df_err)? .metadata; - self.table.commit(metadata).await?; - Ok(data_files) + self.commit(metadata).await } } +fn next_snapshot_id(metadata: &TableMetadata) -> i64 { + metadata + .snapshots() + .map(|s| s.snapshot_id()) + .max() + .unwrap_or(0) + + 1 +} + /// Returns `batch` laid out as `schema`: its columns matched by name, put in /// schema order, cast to the schema's types and tagged with its field ids. /// Missing columns are null. diff --git a/crates/test-tools/src/storage.rs b/crates/test-tools/src/storage.rs index faeaa62..48b06d2 100644 --- a/crates/test-tools/src/storage.rs +++ b/crates/test-tools/src/storage.rs @@ -15,9 +15,10 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::ops::Range; -use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; use bytes::Bytes; @@ -39,9 +40,16 @@ use serde::{Deserialize, Serialize}; pub struct ReadCountingStorageFactory { #[serde(skip)] reads: Arc, + #[serde(skip)] + opens: Arc>>, } impl ReadCountingStorageFactory { + /// Number of readers opened for a particular Parquet file, independent of range reads. + pub fn file_opens(&self, path: &str) -> usize { + self.opens.lock().unwrap().get(path).copied().unwrap_or(0) + } + /// Returns how many byte ranges have been read from data files since the /// last call, and resets the count. pub fn take_reads(&self) -> usize { @@ -54,6 +62,7 @@ impl StorageFactory for ReadCountingStorageFactory { fn build(&self, _config: &StorageConfig) -> Result> { Ok(Arc::new(ReadCountingStorage { reads: self.reads.clone(), + opens: self.opens.clone(), })) } } @@ -62,6 +71,8 @@ impl StorageFactory for ReadCountingStorageFactory { struct ReadCountingStorage { #[serde(skip)] reads: Arc, + #[serde(skip)] + opens: Arc>>, } #[async_trait] @@ -80,6 +91,12 @@ impl Storage for ReadCountingStorage { } async fn reader(&self, path: &str) -> Result> { + *self + .opens + .lock() + .unwrap() + .entry(path.to_owned()) + .or_default() += 1; Ok(Box::new(CountingFileRead { inner: LocalFsStorage::new().reader(path).await?, reads: self.reads.clone(),