From a16ffe709c258595895f5d3c9f00d34b3abedc9f Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 13:55:04 -0500 Subject: [PATCH 1/9] perf(arrow): use shared membership sets for large IN predicates --- .../src/arrow/reader/predicate_visitor.rs | 239 +++++++++++++----- crates/iceberg/src/expr/predicate.rs | 133 ++++++++-- 2 files changed, 285 insertions(+), 87 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 264c34a9a3..dafe66eaf3 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -24,7 +24,7 @@ use std::sync::Arc; use arrow_arith::boolean::{and, and_kleene, is_not_null, is_null, not, or, or_kleene}; use arrow_array::cast::AsArray; -use arrow_array::types::{Float32Type, Float64Type}; +use arrow_array::types::{Float32Type, Float64Type, Int32Type, Int64Type}; use arrow_array::{Array, ArrayRef, BooleanArray, Datum as ArrowDatum, RecordBatch, Scalar}; use arrow_buffer::BooleanBuffer; use arrow_cast::cast::cast; @@ -32,12 +32,13 @@ use arrow_ord::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; use arrow_schema::{ArrowError, DataType}; use arrow_string::like::starts_with; use fnv::FnvHashSet; +use once_cell::unsync::OnceCell; use parquet::schema::types::SchemaDescriptor; use crate::arrow::get_arrow_datum; use crate::error::{Result, invalid_data}; use crate::expr::visitors::bound_predicate_visitor::BoundPredicateVisitor; -use crate::expr::{BoundPredicate, BoundReference}; +use crate::expr::{BoundPredicate, BoundReference, SetIndex}; use crate::spec::Datum; /// A visitor to collect field ids from bound predicates. @@ -236,6 +237,58 @@ impl PredicateConverter<'_> { } } + fn set_predicate( + &mut self, + reference: &BoundReference, + predicate: &BoundPredicate, + negated: bool, + ) -> Result> { + let Some(idx) = self.bound_reference(reference)? else { + return Ok(Box::new(move |batch| { + Ok(constant_bool_array(negated, batch.num_rows())) + })); + }; + let BoundPredicate::Set(set) = predicate else { + unreachable!("set visitor receives a set predicate") + }; + let index = set.membership_index(); + let set = set.clone(); + let arrow_literals = move || { + set.literals() + .iter() + .map(get_arrow_datum) + .collect::>>() + }; + let scalars = if index.is_some() { + OnceCell::new() + } else { + OnceCell::with_value(arrow_literals()?) + }; + Ok(Box::new(move |batch| { + let column = project_column(&batch, idx)?; + if let Some(result) = index + .as_ref() + .and_then(|index| probe_set(index, &column, negated)) + { + return Ok(result); + } + // Schema evolution can give a file a different physical type from the bound literals. + let scalars = scalars + .get_or_try_init(&arrow_literals) + .map_err(|err| ArrowError::ExternalError(Box::new(err)))?; + let mut acc = constant_bool_array(negated, batch.num_rows()); + for scalar in scalars { + let scalar = try_cast_literal(scalar, column.data_type())?; + acc = if negated { + and(&acc, &neq(&column, scalar.as_ref())?)? + } else { + or(&acc, &eq(&column, scalar.as_ref())?)? + }; + } + Ok(acc) + })) + } + /// Build an Arrow predicate that always returns true. fn build_always_true(&self) -> Result> { Ok(Box::new(|batch| { @@ -278,6 +331,31 @@ fn project_column( } } +fn probe_set(index: &SetIndex, column: &ArrayRef, negated: bool) -> Option { + Some(match (index, column.data_type()) { + (SetIndex::Int(keys), DataType::Int32) => { + BooleanArray::from_unary(column.as_primitive::(), |x| { + keys.contains(&x) ^ negated + }) + } + (SetIndex::Long(keys), DataType::Int64) => { + BooleanArray::from_unary(column.as_primitive::(), |x| { + keys.contains(&x) ^ negated + }) + } + (SetIndex::String(keys), DataType::Utf8) => { + BooleanArray::from_unary(column.as_string::(), |s| keys.contains(s) ^ negated) + } + (SetIndex::String(keys), DataType::LargeUtf8) => { + BooleanArray::from_unary(column.as_string::(), |s| keys.contains(s) ^ negated) + } + (SetIndex::String(keys), DataType::Utf8View) => { + BooleanArray::from_unary(column.as_string_view(), |s| keys.contains(s) ^ negated) + } + _ => return None, + }) +} + fn compute_is_nan(array: &ArrayRef) -> std::result::Result { // Compute NaN over the contiguous values slice, then fold the null bitmap // in with a single bitwise AND so that null slots become false. @@ -581,60 +659,19 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { fn r#in( &mut self, reference: &BoundReference, - literals: &FnvHashSet, - _predicate: &BoundPredicate, + _literals: &FnvHashSet, + predicate: &BoundPredicate, ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literals: Vec<_> = literals - .iter() - .map(|lit| get_arrow_datum(lit).unwrap()) - .collect(); - - Ok(Box::new(move |batch| { - // update this if arrow ever adds a native is_in kernel - let left = project_column(&batch, idx)?; - - let mut acc = constant_bool_array(false, batch.num_rows()); - for literal in &literals { - let literal = try_cast_literal(literal, left.data_type())?; - acc = or(&acc, &eq(&left, literal.as_ref())?)? - } - - Ok(acc) - })) - } else { - // A missing column, treating it as null. - self.build_always_false() - } + self.set_predicate(reference, predicate, false) } fn not_in( &mut self, reference: &BoundReference, - literals: &FnvHashSet, - _predicate: &BoundPredicate, + _literals: &FnvHashSet, + predicate: &BoundPredicate, ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literals: Vec<_> = literals - .iter() - .map(|lit| get_arrow_datum(lit).unwrap()) - .collect(); - - Ok(Box::new(move |batch| { - // update this if arrow ever adds a native not_in kernel - let left = project_column(&batch, idx)?; - let mut acc = constant_bool_array(true, batch.num_rows()); - for literal in &literals { - let literal = try_cast_literal(literal, left.data_type())?; - acc = and(&acc, &neq(&left, literal.as_ref())?)? - } - - Ok(acc) - })) - } else { - // A missing column, treating it as null. - self.build_always_true() - } + self.set_predicate(reference, predicate, true) } } @@ -664,15 +701,14 @@ mod tests { use std::collections::{HashMap, HashSet}; use std::sync::Arc; - use arrow_array::{Array, BooleanArray, RecordBatch}; + use arrow_array::{Array, ArrayRef, BooleanArray, RecordBatch}; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; - use parquet::schema::parser::parse_message_type; - use parquet::schema::types::SchemaDescriptor; + use parquet::arrow::ArrowSchemaConverter; use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::{Bind, Predicate, Reference}; - use crate::spec::{NestedField, PrimitiveType, Schema, SchemaRef, Type}; + use crate::spec::{Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type}; fn table_schema_simple() -> SchemaRef { Arc::new( @@ -767,18 +803,16 @@ mod tests { schema: SchemaRef, batch: RecordBatch, ) -> BooleanArray { + let id = schema + .field_by_name(batch.schema().field(0).name()) + .unwrap() + .id; let bound = predicate.bind(schema, true).unwrap(); - // Build a trivial Parquet schema with one float column at field id 4 - let message_type = " - message schema { - optional float qux = 4; - } - "; - let parquet_type = parse_message_type(message_type).expect("parse schema"); - let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); - - let column_map = HashMap::from([(4i32, 0usize)]); + let parquet_schema = ArrowSchemaConverter::new() + .convert(&batch.schema()) + .unwrap(); + let column_map = HashMap::from([(id, 0)]); let column_indices = vec![0usize]; let mut converter = PredicateConverter { @@ -791,6 +825,85 @@ mod tests { predicate_fn(batch).unwrap() } + fn check_set_filter(column: ArrayRef, ty: PrimitiveType, literals: Vec) { + let schema = Arc::new( + Schema::builder() + .with_fields([NestedField::optional(1, "key", Type::Primitive(ty)).into()]) + .build() + .unwrap(), + ); + let batch = RecordBatch::try_from_iter([("key", column)]) + .unwrap() + .slice(1, 4); + let key = Reference::new("key"); + let matches = [Some(true), None, Some(false), Some(true)]; + assert_eq!( + apply_predicate_to_batch( + key.clone().is_in(literals.clone()), + schema.clone(), + batch.clone() + ), + BooleanArray::from_iter(matches) + ); + assert_eq!( + apply_predicate_to_batch(key.is_not_in(literals), schema, batch), + BooleanArray::from_iter(matches.map(|v| v.map(|matched| !matched))) + ); + } + + #[test] + fn integer_set_filters_preserve_nulls_and_physical_casts() { + use arrow_array::{Int32Array, Int64Array}; + + let values = [Some(-1), Some(0), None, Some(-1), Some(63)]; + let ints: ArrayRef = Arc::new(Int32Array::from_iter(values)); + check_set_filter( + ints.clone(), + PrimitiveType::Int, + (0..64).map(Datum::int).collect(), + ); + let longs: ArrayRef = Arc::new(Int64Array::from_iter(values.map(|v| v.map(i64::from)))); + check_set_filter( + longs, + PrimitiveType::Long, + (0..64).map(Datum::long).collect(), + ); + // An older file can still contain Int32 after the table column is promoted. + check_set_filter( + ints, + PrimitiveType::Long, + (0..64).map(Datum::long).collect(), + ); + } + + #[test] + fn string_set_filters_match_across_arrow_encodings() { + use arrow_array::StringArray; + use arrow_cast::cast::cast; + + let strings: ArrayRef = Arc::new(StringArray::from(vec![ + None, + Some(""), + None, + Some("another long string"), + Some("a long string outside the view"), + ])); + let literals = (0..64) + .map(|i| Datum::string(i.to_string())) + .chain([ + Datum::string(""), + Datum::string("a long string outside the view"), + ]) + .collect::>(); + for ty in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { + check_set_filter( + cast(&strings, &ty).unwrap(), + PrimitiveType::String, + literals.clone(), + ); + } + } + #[test] fn test_predicate_converter_nan() { use arrow_array::Float32Array; diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index f8ea896395..de2f0bfa43 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -21,6 +21,7 @@ use std::fmt::{Debug, Display, Formatter}; use std::ops::Not; +use std::sync::{Arc, OnceLock}; use array_init::array_init; use fnv::FnvHashSet; @@ -32,7 +33,7 @@ use crate::expr::visitors::bound_predicate_visitor::visit as visit_bound; use crate::expr::visitors::predicate_visitor::visit; use crate::expr::visitors::rewrite_not::RewriteNotVisitor; use crate::expr::{Bind, BoundReference, PredicateOperator, Reference}; -use crate::spec::{Datum, PrimitiveLiteral, SchemaRef}; +use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType, SchemaRef}; use crate::{Error, ErrorKind}; /// Logical expression, such as `AND`, `OR`, `NOT`. @@ -245,7 +246,55 @@ pub struct SetExpression { /// Term of this predicate, for example, `a` in `a in (1, 2, 3)`. term: T, /// Literals of this predicate, for example, `(1, 2, 3)` in `a in (1, 2, 3)`. - literals: FnvHashSet, + literals: Arc, +} + +// File tasks clone the bound predicate. Keep the lookup with its immutable literals +// so every reader using that predicate can share the same index. +#[derive(Serialize, Deserialize)] +#[serde(transparent)] +struct SetLiterals { + values: FnvHashSet, + #[serde(skip)] + index: OnceLock>>, +} + +impl PartialEq for SetLiterals { + fn eq(&self, other: &Self) -> bool { + self.values == other.values + } +} + +pub(crate) enum SetIndex { + Int(FnvHashSet), + Long(FnvHashSet), + String(FnvHashSet), +} + +impl SetIndex { + fn new(values: &FnvHashSet) -> Option { + let mut index = match values.iter().next()?.data_type() { + PrimitiveType::Int => Self::Int(FnvHashSet::default()), + PrimitiveType::Long => Self::Long(FnvHashSet::default()), + PrimitiveType::String => Self::String(FnvHashSet::default()), + _ => return None, + }; + for datum in values { + match (&mut index, datum.data_type(), datum.literal()) { + (Self::Int(set), PrimitiveType::Int, PrimitiveLiteral::Int(value)) => { + set.insert(*value); + } + (Self::Long(set), PrimitiveType::Long, PrimitiveLiteral::Long(value)) => { + set.insert(*value); + } + (Self::String(set), PrimitiveType::String, PrimitiveLiteral::String(value)) => { + set.insert(value.clone()); + } + _ => return None, + } + } + Some(index) + } } impl Debug for SetExpression { @@ -253,7 +302,7 @@ impl Debug for SetExpression { f.debug_struct("SetExpression") .field("op", &self.op) .field("term", &self.term) - .field("literal", &self.literals) + .field("literal", self.literals()) .finish() } } @@ -276,7 +325,14 @@ impl SetExpression { /// ``` pub fn new(op: PredicateOperator, term: T, literals: FnvHashSet) -> Self { debug_assert!(op.is_set()); - Self { op, term, literals } + Self { + op, + term, + literals: Arc::new(SetLiterals { + values: literals, + index: OnceLock::new(), + }), + } } /// Return the operator of this predicate. @@ -286,7 +342,19 @@ impl SetExpression { /// Return the hash set of values compared against the term in this expression. pub fn literals(&self) -> &FnvHashSet { - &self.literals + &self.literals.values + } + + pub(crate) fn membership_index(&self) -> Option> { + // Short lists benefit from Arrow's vectorized scalar comparisons. + const MIN_INDEX_SIZE: usize = 32; + if self.literals().len() < MIN_INDEX_SIZE { + return None; + } + self.literals + .index + .get_or_init(|| SetIndex::new(self.literals()).map(Arc::new)) + .clone() } /// Return the term of this predicate. @@ -300,17 +368,17 @@ impl Bind for SetExpression { fn bind(&self, schema: SchemaRef, case_sensitive: bool) -> Result { let bound_term = self.term.bind(schema.clone(), case_sensitive)?; - Ok(SetExpression::new( - self.op, - bound_term, - self.literals.clone(), - )) + Ok(SetExpression { + op: self.op, + term: bound_term, + literals: Arc::clone(&self.literals), + }) } } impl Display for SetExpression { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut literal_strs = self.literals.iter().map(|l| format!("{l}")); + let mut literal_strs = self.literals().iter().map(|l| format!("{l}")); write!(f, "{} {} ({})", self.term, self.op, literal_strs.join(", ")) } @@ -457,8 +525,9 @@ impl Bind for Predicate { Predicate::Set(expr) => { let bound_expr = expr.bind(schema, case_sensitive)?; let bound_literals = bound_expr - .literals - .into_iter() + .literals() + .iter() + .cloned() .map(|l| l.to(&bound_expr.term.field().field_type)) .collect::>>()?; @@ -640,11 +709,10 @@ impl Predicate { expr.term, expr.literal, )), - Predicate::Set(expr) => Predicate::Set(SetExpression::new( - expr.op.negate(), - expr.term, - expr.literals, - )), + Predicate::Set(expr) => Predicate::Set(SetExpression { + op: expr.op.negate(), + ..expr + }), } } /// Simplifies the expression by removing `NOT` predicates, @@ -751,11 +819,10 @@ impl BoundPredicate { expr.term, expr.literal, )), - BoundPredicate::Set(expr) => BoundPredicate::Set(SetExpression::new( - expr.op.negate(), - expr.term, - expr.literals, - )), + BoundPredicate::Set(expr) => BoundPredicate::Set(SetExpression { + op: expr.op.negate(), + ..expr + }), } } @@ -820,9 +887,27 @@ mod tests { use std::sync::Arc; use crate::expr::Predicate::{AlwaysFalse, AlwaysTrue}; - use crate::expr::{Bind, BoundPredicate, Reference}; + use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; use crate::spec::{Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type}; + #[test] + fn cloned_set_predicates_share_the_index_and_serialize_as_literals() { + let predicate = Reference::new("key").is_in((0..64).map(Datum::int)); + let encoded = serde_json::to_value(&predicate).unwrap(); + assert_eq!(encoded["Set"]["literals"].as_array().unwrap().len(), 64); + let restored: Predicate = serde_json::from_value(encoded.clone()).unwrap(); + let clone = predicate.clone().not().rewrite_not(); + let (Predicate::Set(original), Predicate::Set(negated)) = (&predicate, &clone) else { + panic!("expected set predicates"); + }; + assert!(Arc::ptr_eq( + &original.membership_index().unwrap(), + &negated.membership_index().unwrap() + )); + assert_eq!(predicate, restored); + assert_eq!(serde_json::to_value(&predicate).unwrap(), encoded); + } + #[test] fn test_logical_or_rewrite_not() { let expression = Reference::new("b") From 0c658b41e4a7df48f3362aa21056783a87b05696 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 15:16:45 -0500 Subject: [PATCH 2/9] perf(arrow): use foldhash for string IN lookups --- Cargo.lock | 9 ++++++++- Cargo.toml | 1 + crates/iceberg/Cargo.toml | 1 + crates/iceberg/src/expr/predicate.rs | 6 ++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbf366a32f..2b2400f6bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2154,6 +2154,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2505,7 +2511,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -2860,6 +2866,7 @@ dependencies = [ "fastnum", "flate2", "fnv", + "foldhash 0.2.0", "form_urlencoded", "futures", "iceberg-property-macro", diff --git a/Cargo.toml b/Cargo.toml index 1660c8c521..5bb99183bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,6 +94,7 @@ fastnum = { version = "0.7", default-features = false, features = [ faststr = "0.2.31" flate2 = "1.1.5" fnv = "1.0.7" +foldhash = "0.2" form_urlencoded = "1.2.2" futures = "0.3" hive_metastore = "0.2.0" diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 45e0cae412..e4dfa79bab 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -58,6 +58,7 @@ derive_builder = { workspace = true } fastnum = { workspace = true } flate2 = { workspace = true } fnv = { workspace = true } +foldhash = { workspace = true } form_urlencoded = { workspace = true } futures = { workspace = true } iceberg-property-macro = { workspace = true } diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index de2f0bfa43..44a4feb184 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -19,12 +19,14 @@ //! Predicate expressions are used to filter data, and evaluates to a boolean value. For example, //! `a > 10` is a predicate expression, and it evaluates to `true` if `a` is greater than `10`, +use std::collections::HashSet; use std::fmt::{Debug, Display, Formatter}; use std::ops::Not; use std::sync::{Arc, OnceLock}; use array_init::array_init; use fnv::FnvHashSet; +use foldhash::fast::FixedState; use itertools::Itertools; use serde::{Deserialize, Serialize}; @@ -268,7 +270,7 @@ impl PartialEq for SetLiterals { pub(crate) enum SetIndex { Int(FnvHashSet), Long(FnvHashSet), - String(FnvHashSet), + String(HashSet), } impl SetIndex { @@ -276,7 +278,7 @@ impl SetIndex { let mut index = match values.iter().next()?.data_type() { PrimitiveType::Int => Self::Int(FnvHashSet::default()), PrimitiveType::Long => Self::Long(FnvHashSet::default()), - PrimitiveType::String => Self::String(FnvHashSet::default()), + PrimitiveType::String => Self::String(HashSet::default()), _ => return None, }; for datum in values { From 1ca9b4bf7bdbd399d0f3193e652ce44217aa74ca Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 17:00:46 -0500 Subject: [PATCH 3/9] perf(arrow): probe dictionary values for IN predicates Review: https://github.com/apache/iceberg-rust/pull/3739#discussion_r4648112922 --- .../src/arrow/reader/predicate_visitor.rs | 59 +++++++++++++++---- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index dafe66eaf3..cf0074520f 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -30,6 +30,7 @@ use arrow_buffer::BooleanBuffer; use arrow_cast::cast::cast; use arrow_ord::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; use arrow_schema::{ArrowError, DataType}; +use arrow_select::take::take; use arrow_string::like::starts_with; use fnv::FnvHashSet; use once_cell::unsync::OnceCell; @@ -266,9 +267,8 @@ impl PredicateConverter<'_> { }; Ok(Box::new(move |batch| { let column = project_column(&batch, idx)?; - if let Some(result) = index - .as_ref() - .and_then(|index| probe_set(index, &column, negated)) + if let Some(index) = &index + && let Some(result) = probe_set(index, column.as_ref(), negated)? { return Ok(result); } @@ -331,8 +331,18 @@ fn project_column( } } -fn probe_set(index: &SetIndex, column: &ArrayRef, negated: bool) -> Option { - Some(match (index, column.data_type()) { +fn probe_set( + index: &SetIndex, + column: &dyn Array, + negated: bool, +) -> std::result::Result, ArrowError> { + if let Some(dict) = column.as_any_dictionary_opt() { + let Some(values) = probe_set(index, dict.values().as_ref(), negated)? else { + return Ok(None); + }; + return Ok(Some(take(&values, dict.keys(), None)?.as_boolean().clone())); + } + Ok(Some(match (index, column.data_type()) { (SetIndex::Int(keys), DataType::Int32) => { BooleanArray::from_unary(column.as_primitive::(), |x| { keys.contains(&x) ^ negated @@ -352,8 +362,8 @@ fn probe_set(index: &SetIndex, column: &ArrayRef, negated: bool) -> Option { BooleanArray::from_unary(column.as_string_view(), |s| keys.contains(s) ^ negated) } - _ => return None, - }) + _ => return Ok(None), + })) } fn compute_is_nan(array: &ArrayRef) -> std::result::Result { @@ -705,9 +715,9 @@ mod tests { use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use parquet::arrow::ArrowSchemaConverter; - use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; + use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array, probe_set}; use crate::expr::visitors::bound_predicate_visitor::visit; - use crate::expr::{Bind, Predicate, Reference}; + use crate::expr::{Bind, Predicate, Reference, SetIndex}; use crate::spec::{Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type}; fn table_schema_simple() -> SchemaRef { @@ -895,7 +905,12 @@ mod tests { Datum::string("a long string outside the view"), ]) .collect::>(); - for ty in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { + for ty in [ + DataType::Utf8, + DataType::LargeUtf8, + DataType::Utf8View, + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + ] { check_set_filter( cast(&strings, &ty).unwrap(), PrimitiveType::String, @@ -904,6 +919,30 @@ mod tests { } } + #[test] + fn dictionary_set_probes_preserve_null_keys_and_values() { + use arrow_array::types::Int32Type; + use arrow_array::{DictionaryArray, Int32Array, StringArray}; + + let values = Arc::new(StringArray::from(vec![ + Some("hit"), + None, + Some("miss"), + Some("unused"), + ])); + let keys = Int32Array::from(vec![Some(0), Some(2), None, Some(1), Some(0)]); + let dictionary = DictionaryArray::::try_new(keys, values).unwrap(); + let index = SetIndex::String([String::from("hit")].into_iter().collect()); + let matches = [Some(true), Some(false), None, None, Some(true)]; + for negated in [false, true] { + let expected = BooleanArray::from_iter(matches.map(|v| v.map(|hit| hit ^ negated))); + assert_eq!( + probe_set(&index, &dictionary, negated).unwrap(), + Some(expected) + ); + } + } + #[test] fn test_predicate_converter_nan() { use arrow_array::Float32Array; From 3c20cdf677af71048d9bac5789d9a9f1a56d9d9c Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 17:00:48 -0500 Subject: [PATCH 4/9] test(arrow): assert which IN column types use the index Review: https://github.com/apache/iceberg-rust/pull/3739#discussion_r4648114820 --- .../src/arrow/reader/predicate_visitor.rs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index cf0074520f..95f7f28405 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -919,6 +919,30 @@ mod tests { } } + #[test] + fn set_index_probes_supported_column_types() { + use arrow_array::{Int32Array, Int64Array, StringArray}; + use arrow_cast::cast::cast; + + let ints = Int32Array::from(vec![1, 2]); + let longs = Int64Array::from(vec![1, 2]); + let strings = StringArray::from(vec!["1", "2"]); + let int_index = SetIndex::Int([1].into_iter().collect()); + let long_index = SetIndex::Long([1].into_iter().collect()); + let string_index = SetIndex::String([String::from("1")].into_iter().collect()); + let expected = Some(BooleanArray::from(vec![true, false])); + assert_eq!(probe_set(&int_index, &ints, false).unwrap(), expected); + assert_eq!(probe_set(&long_index, &longs, false).unwrap(), expected); + assert_eq!(probe_set(&long_index, &ints, false).unwrap(), None); + for ty in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { + let column = cast(&strings, &ty).unwrap(); + assert_eq!( + probe_set(&string_index, column.as_ref(), false).unwrap(), + expected + ); + } + } + #[test] fn dictionary_set_probes_preserve_null_keys_and_values() { use arrow_array::types::Int32Type; From eb8026f0b5ec3f2d7ab2c38df897adec15fa6da9 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 17:00:48 -0500 Subject: [PATCH 5/9] test(expr): check bound IN index sharing through NOT rewrites Review: https://github.com/apache/iceberg-rust/pull/3739#discussion_r4648119051 --- crates/iceberg/src/expr/predicate.rs | 31 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index 44a4feb184..2b15bd8bc4 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -889,25 +889,36 @@ mod tests { use std::sync::Arc; use crate::expr::Predicate::{AlwaysFalse, AlwaysTrue}; - use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; + use crate::expr::{Bind, BoundPredicate, LogicalExpression, PredicateOperator, Reference}; use crate::spec::{Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type}; #[test] fn cloned_set_predicates_share_the_index_and_serialize_as_literals() { - let predicate = Reference::new("key").is_in((0..64).map(Datum::int)); + let predicate = Reference::new("bar") + .is_in((0..64).map(Datum::int)) + .bind(table_schema_simple(), true) + .unwrap(); let encoded = serde_json::to_value(&predicate).unwrap(); assert_eq!(encoded["Set"]["literals"].as_array().unwrap().len(), 64); - let restored: Predicate = serde_json::from_value(encoded.clone()).unwrap(); - let clone = predicate.clone().not().rewrite_not(); - let (Predicate::Set(original), Predicate::Set(negated)) = (&predicate, &clone) else { + let BoundPredicate::Set(original) = &predicate else { panic!("expected set predicates"); }; - assert!(Arc::ptr_eq( - &original.membership_index().unwrap(), - &negated.membership_index().unwrap() - )); - assert_eq!(predicate, restored); + let index = original.membership_index().unwrap(); + let negated = BoundPredicate::Not(LogicalExpression::new([Box::new(predicate.clone())])) + .rewrite_not(); + let BoundPredicate::Set(negated) = negated else { + panic!("expected a negated set predicate"); + }; + assert_eq!(negated.op(), PredicateOperator::NotIn); + assert!(Arc::ptr_eq(&index, &negated.membership_index().unwrap())); + assert_eq!(serde_json::to_value(&predicate).unwrap(), encoded); + let restored: BoundPredicate = serde_json::from_value(encoded).unwrap(); + assert_eq!(predicate, restored); + let BoundPredicate::Set(restored) = restored else { + panic!("expected a restored set predicate"); + }; + assert!(!Arc::ptr_eq(&index, &restored.membership_index().unwrap())); } #[test] From 6c9ea06589f1ca02e3f000e701327ea576a8624f Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 17:03:14 -0500 Subject: [PATCH 6/9] perf(arrow): use foldhash for integer IN indexes Review: https://github.com/apache/iceberg-rust/pull/3739#discussion_r4648107743 --- crates/iceberg/src/expr/predicate.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index 2b15bd8bc4..4e6f71b0ca 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -268,16 +268,16 @@ impl PartialEq for SetLiterals { } pub(crate) enum SetIndex { - Int(FnvHashSet), - Long(FnvHashSet), + Int(HashSet), + Long(HashSet), String(HashSet), } impl SetIndex { fn new(values: &FnvHashSet) -> Option { let mut index = match values.iter().next()?.data_type() { - PrimitiveType::Int => Self::Int(FnvHashSet::default()), - PrimitiveType::Long => Self::Long(FnvHashSet::default()), + PrimitiveType::Int => Self::Int(HashSet::default()), + PrimitiveType::Long => Self::Long(HashSet::default()), PrimitiveType::String => Self::String(HashSet::default()), _ => return None, }; From 63ebab2e493ad2d6ea87246b22893cd88ad698a6 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 17:05:19 -0500 Subject: [PATCH 7/9] perf(arrow): choose IN index cutoffs in the reader by type Use the first tested list size where the foldhash index improves the scan over both one and 64 files: four string keys, eight Int32 keys, or sixteen Int64 keys. Review: https://github.com/apache/iceberg-rust/pull/3739#discussion_r4648117006 --- .../iceberg/src/arrow/reader/predicate_visitor.rs | 13 +++++++++++-- crates/iceberg/src/expr/predicate.rs | 5 ----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 95f7f28405..594d2b0b3c 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -40,7 +40,7 @@ use crate::arrow::get_arrow_datum; use crate::error::{Result, invalid_data}; use crate::expr::visitors::bound_predicate_visitor::BoundPredicateVisitor; use crate::expr::{BoundPredicate, BoundReference, SetIndex}; -use crate::spec::Datum; +use crate::spec::{Datum, PrimitiveType}; /// A visitor to collect field ids from bound predicates. pub(super) struct CollectFieldIdVisitor { @@ -252,7 +252,16 @@ impl PredicateConverter<'_> { let BoundPredicate::Set(set) = predicate else { unreachable!("set visitor receives a set predicate") }; - let index = set.membership_index(); + // The scan benchmark crosses over from scalar comparisons to hashing at different list sizes. + let index = match ( + reference.field().field_type.as_primitive_type(), + set.literals().len(), + ) { + (Some(PrimitiveType::String), 4..) + | (Some(PrimitiveType::Int), 8..) + | (Some(PrimitiveType::Long), 16..) => set.membership_index(), + _ => None, + }; let set = set.clone(); let arrow_literals = move || { set.literals() diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index 4e6f71b0ca..eef1e1019b 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -348,11 +348,6 @@ impl SetExpression { } pub(crate) fn membership_index(&self) -> Option> { - // Short lists benefit from Arrow's vectorized scalar comparisons. - const MIN_INDEX_SIZE: usize = 32; - if self.literals().len() < MIN_INDEX_SIZE { - return None; - } self.literals .index .get_or_init(|| SetIndex::new(self.literals()).map(Arc::new)) From 25daa58277177674f939fca86df4359d466cd878 Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 17:31:02 -0500 Subject: [PATCH 8/9] test(arrow): probe indexes built from bound IN predicates Exercise index construction for each literal type before probing the Arrow column, including dictionary-encoded strings. Review: https://github.com/apache/iceberg-rust/pull/3739#discussion_r4648114820 --- .../src/arrow/reader/predicate_visitor.rs | 44 ++++++++++++++----- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 594d2b0b3c..9ffe18dd9b 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -726,7 +726,7 @@ mod tests { use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array, probe_set}; use crate::expr::visitors::bound_predicate_visitor::visit; - use crate::expr::{Bind, Predicate, Reference, SetIndex}; + use crate::expr::{Bind, BoundPredicate, Predicate, Reference, SetIndex}; use crate::spec::{Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type}; fn table_schema_simple() -> SchemaRef { @@ -844,13 +844,17 @@ mod tests { predicate_fn(batch).unwrap() } - fn check_set_filter(column: ArrayRef, ty: PrimitiveType, literals: Vec) { - let schema = Arc::new( + fn key_schema(ty: PrimitiveType) -> SchemaRef { + Arc::new( Schema::builder() .with_fields([NestedField::optional(1, "key", Type::Primitive(ty)).into()]) .build() .unwrap(), - ); + ) + } + + fn check_set_filter(column: ArrayRef, ty: PrimitiveType, literals: Vec) { + let schema = key_schema(ty); let batch = RecordBatch::try_from_iter([("key", column)]) .unwrap() .slice(1, 4); @@ -933,17 +937,35 @@ mod tests { use arrow_array::{Int32Array, Int64Array, StringArray}; use arrow_cast::cast::cast; - let ints = Int32Array::from(vec![1, 2]); - let longs = Int64Array::from(vec![1, 2]); - let strings = StringArray::from(vec!["1", "2"]); - let int_index = SetIndex::Int([1].into_iter().collect()); - let long_index = SetIndex::Long([1].into_iter().collect()); - let string_index = SetIndex::String([String::from("1")].into_iter().collect()); + let index = |ty, literals: Vec| { + let BoundPredicate::Set(set) = Reference::new("key") + .is_in(literals) + .bind(key_schema(ty), true) + .unwrap() + else { + panic!("expected a bound set predicate"); + }; + set.membership_index().unwrap() + }; + let int_index = index(PrimitiveType::Int, (0..64).map(Datum::int).collect()); + let long_index = index(PrimitiveType::Long, (0..64).map(Datum::long).collect()); + let string_index = index( + PrimitiveType::String, + (0..64).map(|i| Datum::string(i.to_string())).collect(), + ); + let ints = Int32Array::from(vec![1, 100]); + let longs = Int64Array::from(vec![1, 100]); + let strings = StringArray::from(vec!["1", "100"]); let expected = Some(BooleanArray::from(vec![true, false])); assert_eq!(probe_set(&int_index, &ints, false).unwrap(), expected); assert_eq!(probe_set(&long_index, &longs, false).unwrap(), expected); assert_eq!(probe_set(&long_index, &ints, false).unwrap(), None); - for ty in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { + for ty in [ + DataType::Utf8, + DataType::LargeUtf8, + DataType::Utf8View, + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + ] { let column = cast(&strings, &ty).unwrap(); assert_eq!( probe_set(&string_index, column.as_ref(), false).unwrap(), From 5c7d3f6afb7227b49b575629d926d3a7d3f5e84b Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Fri, 20 Nov 2026 17:31:04 -0500 Subject: [PATCH 9/9] test(expr): clone bound IN predicates before building the index File tasks can clone and rewrite a predicate before its first lookup. Require both cold clones to initialize the same index. Review: https://github.com/apache/iceberg-rust/pull/3739#discussion_r4648119051 --- crates/iceberg/src/expr/predicate.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index eef1e1019b..b168831afe 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -895,16 +895,14 @@ mod tests { .unwrap(); let encoded = serde_json::to_value(&predicate).unwrap(); assert_eq!(encoded["Set"]["literals"].as_array().unwrap().len(), 64); - let BoundPredicate::Set(original) = &predicate else { - panic!("expected set predicates"); - }; - let index = original.membership_index().unwrap(); let negated = BoundPredicate::Not(LogicalExpression::new([Box::new(predicate.clone())])) .rewrite_not(); - let BoundPredicate::Set(negated) = negated else { - panic!("expected a negated set predicate"); + let (BoundPredicate::Set(original), BoundPredicate::Set(negated)) = (&predicate, &negated) + else { + panic!("expected set predicates"); }; assert_eq!(negated.op(), PredicateOperator::NotIn); + let index = original.membership_index().unwrap(); assert!(Arc::ptr_eq(&index, &negated.membership_index().unwrap())); assert_eq!(serde_json::to_value(&predicate).unwrap(), encoded);