diff --git a/Cargo.lock b/Cargo.lock index b111a48..264557d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,6 +87,15 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -102,6 +111,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -1141,6 +1156,12 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.4.3" @@ -1200,6 +1221,33 @@ dependencies = [ "phf", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -1413,6 +1461,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -1422,6 +1506,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -3241,6 +3335,7 @@ version = "0.12.0" dependencies = [ "async-trait", "bytes", + "criterion", "dashmap", "datafusion", "futures", @@ -4028,6 +4123,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -4091,6 +4192,16 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking" version = "2.2.1" @@ -4229,6 +4340,34 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polyval" version = "0.6.2" @@ -4510,6 +4649,26 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "recursive" version = "0.1.1" @@ -5376,6 +5535,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.12.0" @@ -5608,7 +5777,7 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.5", + "rand 0.10.4", ] [[package]] @@ -5895,6 +6064,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -5904,6 +6089,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/crates/datafusion/Cargo.toml b/crates/datafusion/Cargo.toml index e7fb281..4281500 100644 --- a/crates/datafusion/Cargo.toml +++ b/crates/datafusion/Cargo.toml @@ -41,6 +41,7 @@ tokio = { workspace = true } uuid = { version = "1.18", features = ["v7"] } [dev-dependencies] +criterion = { version = "0.8", features = ["async_tokio"] } iceberg-datafusion-test-tools = { workspace = true } insta = "1" parquet = "59.2" @@ -52,5 +53,9 @@ tempfile = "3.18" name = "integration" path = "tests/main.rs" +[[bench]] +name = "taxi_in_list" +harness = false + [lints] workspace = true diff --git a/crates/datafusion/benches/taxi_in_list.rs b/crates/datafusion/benches/taxi_in_list.rs new file mode 100644 index 0000000..20291a1 --- /dev/null +++ b/crates/datafusion/benches/taxi_in_list.rs @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end SQL lookup on a taxi table with a high-cardinality string key. +//! Set TAXI_DATA_DIR to use the seven daily NYC taxi Parquet files in place of +//! the built-in taxi fixture. + +use std::env; +use std::error::Error; +use std::fs::File; +use std::path::Path; +use std::sync::Arc; + +use criterion::{BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main}; +use datafusion::arrow::array::{ArrayRef, RecordBatch, RecordBatchReader, StringArray}; +use datafusion::arrow::compute::concat_batches; +use datafusion::arrow::util::display::array_value_to_string; +use datafusion::prelude::{SessionConfig, SessionContext}; +use iceberg::TableIdent; +use iceberg::spec::{NestedField, PrimitiveType, Schema, Transform, Type}; +use iceberg_datafusion::IcebergStaticTableProvider; +use iceberg_datafusion_test_tools::{ + FixtureCatalog, FixtureTable, TAXI_DAYS, TAXI_TRIPS_PER_DAY, taxi_schema, taxi_trips, +}; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; +use tokio::runtime::Builder; + +fn resource_key(row: usize) -> String { + let region = match row % 369 { + 0..14 => "ap-southeast-2", + 14..16 => "eu-central-1", + 16..18 => "eu-south-1", + _ => "us-east-1", + }; + // An odd multiplier scatters adjacent row IDs while keeping the keys unique. + let volume = (row as u64).wrapping_mul(0x9e3779b97f4a7c15); + format!("arn:aws:ec2:{region}:000000000000:volume/vol-{volume:017x}") +} + +async fn taxi_table(data_dir: Option<&Path>) -> Result> { + let catalog = FixtureCatalog::try_new().await?; + let mut fields = taxi_schema().as_struct().fields().to_vec(); + fields.push( + NestedField::required(14, "resource_key", Type::Primitive(PrimitiveType::String)) + .into(), + ); + let mut table = catalog + .table( + TableIdent::from_strs(["default", "taxi"])?, + Schema::builder().with_fields(fields).build()?, + ) + .partition_by("pickup_date", Transform::Identity) + .create() + .await?; + let mut append = table.append().writer_properties( + WriterProperties::builder() + .set_compression(Compression::ZSTD(Default::default())) + .build(), + ); + for day in 0..TAXI_DAYS { + let trips = if let Some(dir) = data_dir { + let path = + dir.join(format!("pickup_date=2024-01-{:02}/data_0.parquet", 8 + day)); + let reader = + ParquetRecordBatchReaderBuilder::try_new(File::open(path)?)?.build()?; + concat_batches(&reader.schema(), &reader.collect::, _>>()?)? + } else { + taxi_trips(day)? + }; + assert_eq!(trips.num_rows(), TAXI_TRIPS_PER_DAY); + let first = day * TAXI_TRIPS_PER_DAY; + let keys = StringArray::from_iter_values( + (first..first + trips.num_rows()).map(resource_key), + ); + let columns = trips + .schema_ref() + .fields() + .iter() + .map(|f| f.name().as_str()) + .zip(trips.columns().iter().cloned()) + .chain([("resource_key", Arc::new(keys) as ArrayRef)]); + append = append.file(RecordBatch::try_from_iter(columns)?); + } + append.commit().await?; + Ok(table) +} + +fn bench_taxi_in_list(c: &mut Criterion) { + let runtime = Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + let data_dir = env::var_os("TAXI_DATA_DIR"); + let table = runtime + .block_on(taxi_table(data_dir.as_deref().map(Path::new))) + .unwrap(); + let ctx = + SessionContext::new_with_config(SessionConfig::new().with_target_partitions(4)); + let provider = + IcebergStaticTableProvider::try_new_from_table(table.table().clone()).unwrap(); + ctx.register_table("taxi", Arc::new(provider)).unwrap(); + + let mut group = c.benchmark_group("taxi_string_in_list"); + group.sample_size(20).sampling_mode(SamplingMode::Flat); + let rows = TAXI_DAYS * TAXI_TRIPS_PER_DAY; + for size in [1, 16, 31, 32, 100, 369, 400, 1000] { + // This stride is coprime to the row count, so the keys span the daily files. + let mut expected: Vec<_> = (0..size) + .map(|i| resource_key((37_417 + i * 104_729) % rows)) + .collect(); + let literals = expected + .iter() + .map(|key| format!("'{key}'")) + .collect::>() + .join(", "); + let sql = + format!("SELECT resource_key FROM taxi WHERE resource_key IN ({literals})"); + let query = || async { ctx.sql(&sql).await.unwrap().collect().await.unwrap() }; + + let mut actual = Vec::new(); + for batch in runtime.block_on(query()) { + for row in 0..batch.num_rows() { + actual.push(array_value_to_string(batch.column(0), row).unwrap()); + } + } + actual.sort_unstable(); + expected.sort_unstable(); + assert_eq!(actual, expected); + + group.bench_function(BenchmarkId::from_parameter(size), |b| { + b.to_async(&runtime).iter(query); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_taxi_in_list); +criterion_main!(benches);