From 813e3706f1d4b2043fd09e69644906bebdb7b4e9 Mon Sep 17 00:00:00 2001 From: Noah Date: Sat, 21 Nov 2026 00:21:55 -0500 Subject: [PATCH] fix: read partition source columns through PartitionExpr children PartitionExpr read its source columns by position in the input batch but reported no children. When ProjectionPushdown merged the partition projection into the projection below it, nothing rewrote those positions, so partition values were computed from the wrong columns. Files landed in the wrong partitions and partition pruning skipped them. PartitionExpr's children are now the table's top-level columns, in table order, and it reads the batch only through them. The optimizer rewrites them like any other column reference. Evaluation lays the children out as the table and still uses iceberg's PartitionValueCalculator. The public `try_new` takes the columns, so a codec passes the decoded children. Closes #35 --- AGENTS.md | 2 +- crates/datafusion/src/write/project.rs | 334 +++++++++++++++++++++---- crates/datafusion/tests/insert_into.rs | 73 +++++- 3 files changed, 354 insertions(+), 55 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1859768..c6df87b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,7 +108,7 @@ input - **`TaskWriter`** picks one of three writers: `UnpartitionedWriter`; `FanoutWriter` (accepts unsorted input, keeps many files open); or `ClusteredWriter` (requires sorted input, which is why the sort step exists). - **Writer limits:** only Parquet is supported. The writer honors `write.parquet.*` table properties and table encryption. It matches columns by name, because DataFusion batches carry no field IDs. -- **`PartitionExpr`** is public and retains its partition spec and table schema so that distributed engines can serialize them and rebuild the expression on workers with `try_new`. Its equality and hash are by value; don't go back to pointer equality. +- **`PartitionExpr`** is public and retains its partition spec and table schema so that distributed engines can serialize them and rebuild the expression on workers with `try_new`, passing the decoded children. Its children are the table's top-level columns, in table order, and it reads the batch only through them, so optimizer rules like `ProjectionPushdown` that move it onto a different input rewrite them like any other column reference. Its equality and hash are by value; don't go back to pointer equality. ### Errors diff --git a/crates/datafusion/src/write/project.rs b/crates/datafusion/src/write/project.rs index c62e4d5..c162f12 100644 --- a/crates/datafusion/src/write/project.rs +++ b/crates/datafusion/src/write/project.rs @@ -20,18 +20,22 @@ use std::sync::Arc; use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema}; +use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; use datafusion::common::plan_err; use datafusion::error::Result; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_expr::expressions::Column; +use datafusion::physical_expr_common::physical_expr::fmt_sql; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::{ColumnarValue, ExecutionPlan}; use iceberg::arrow::{ PROJECTED_PARTITION_VALUE_COLUMN, PartitionValueCalculator, schema_to_arrow_schema, strip_metadata_from_schema, }; -use iceberg::spec::{PartitionSpec, SchemaRef as IcebergSchemaRef}; +use iceberg::spec::{ + NestedField, PartitionSpec, Schema as IcebergSchema, SchemaRef as IcebergSchemaRef, + Type, +}; use iceberg::table::Table; use crate::to_datafusion_error; @@ -91,6 +95,10 @@ pub fn project_with_partition( let partition_expr = Arc::new(PartitionExpr::try_new( partition_spec.clone(), table_schema.clone(), + projection_exprs + .iter() + .map(|(column, _)| Arc::clone(column)) + .collect(), )?); projection_exprs.push((partition_expr, PROJECTED_PARTITION_VALUE_COLUMN.to_string())); @@ -100,30 +108,41 @@ pub fn project_with_partition( /// PhysicalExpr implementation for partition value calculation /// +/// Its children are the table's top-level columns, in table order, and it reads the +/// batch only through them. Optimizer rules that move it onto a different input, +/// such as merging it into the projection below, rewrite them like any other column +/// reference. +/// /// The [`PartitionValueCalculator`] cannot be serialized, so the spec and schema it -/// was built from are retained for [`Self::try_new`] to rebuild from. +/// was built from are retained for [`Self::try_new`] to rebuild from, with the +/// decoded children. #[derive(Debug, Clone)] pub struct PartitionExpr { calculator: Arc, partition_spec: Arc, table_schema: IcebergSchemaRef, + columns: Vec>, } impl PartitionExpr { - /// Builds the expression from the spec and schema that define it. + /// Builds the expression from the spec and schema that define it, reading the + /// table's top-level columns from `columns`, one per column in table order. /// /// The calculator is built here rather than passed in, so it cannot drift from /// the retained inputs. /// /// # Errors /// - /// Returns an error if the spec is unpartitioned or cannot be bound to the schema. + /// Returns an error if the spec is unpartitioned or cannot be bound to the schema, + /// or if `columns` doesn't have one expression per top-level table column. /// /// # Example /// /// ``` /// use std::sync::Arc; /// + /// use datafusion::physical_expr::PhysicalExpr; + /// use datafusion::physical_expr::expressions::Column; /// use iceberg_datafusion::write::PartitionExpr; /// use iceberg::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, Transform, Type}; /// @@ -139,12 +158,14 @@ impl PartitionExpr { /// .add_partition_field("id", "id_bucket", Transform::Bucket(16))? /// .build()?, /// ); - /// let expr = PartitionExpr::try_new(spec, schema)?; + /// let expr = PartitionExpr::try_new(spec, schema, vec![Arc::new(Column::new("id", 0))])?; /// - /// // A worker rebuilds an equal expression from the two retained inputs. + /// // A worker rebuilds an equal expression from the retained spec and schema and + /// // the decoded children. /// let rebuilt = PartitionExpr::try_new( /// Arc::new(expr.partition_spec().as_ref().clone()), /// Arc::new(expr.table_schema().as_ref().clone()), + /// expr.children().into_iter().cloned().collect(), /// )?; /// assert_eq!(expr, rebuilt); /// # Ok::<(), Box>(()) @@ -152,21 +173,25 @@ impl PartitionExpr { pub fn try_new( partition_spec: Arc, table_schema: IcebergSchemaRef, + columns: Vec>, ) -> Result { let calculator = PartitionValueCalculator::try_new( partition_spec.as_ref(), table_schema.as_ref(), ) .map_err(to_datafusion_error)?; + check_column_count(&table_schema, columns.len())?; Ok(Self { calculator: Arc::new(calculator), partition_spec, table_schema, + columns, }) } /// The partition spec this expression computes values for. With - /// [`Self::table_schema`], all [`Self::try_new`] needs to rebuild an equal one. + /// [`Self::table_schema`] and the children, all [`Self::try_new`] needs to + /// rebuild an equal one. pub fn partition_spec(&self) -> &Arc { &self.partition_spec } @@ -176,15 +201,62 @@ impl PartitionExpr { pub fn table_schema(&self) -> &IcebergSchemaRef { &self.table_schema } + + /// Writes each partition field with the column holding its source, e.g. + /// `iceberg_partition_values(id: identity(b@1))`. + fn fmt_with( + &self, + f: &mut std::fmt::Formatter<'_>, + fmt_column: impl Fn(&dyn PhysicalExpr) -> String, + ) -> std::fmt::Result { + let table_columns = self.table_schema.as_struct().fields(); + let fields: Vec = self + .partition_spec + .fields() + .iter() + .map(|field| { + // `try_new` checked that the source is in the schema. + let column = table_columns + .iter() + .position(|column| holds(column, field.source_id)) + .map(|index| fmt_column(self.columns[index].as_ref())) + .unwrap_or_default(); + format!("{}: {}({column})", field.name, field.transform) + }) + .collect(); + write!(f, "iceberg_partition_values({})", fields.join(", ")) + } } -// Equal when the spec and schema are equal. The calculator is derived from them, so -// it takes no part. Both comparisons include `spec_id` and `schema_id`, which `Hash` -// relies on. +/// Whether `field` is the field with `id` or a struct containing it. Partition +/// sources can't be nested in lists or maps. +fn holds(field: &NestedField, id: i32) -> bool { + field.id == id + || matches!( + field.field_type.as_ref(), + Type::Struct(fields) if fields.fields().iter().any(|field| holds(field, id)) + ) +} + +fn check_column_count(table_schema: &IcebergSchema, columns: usize) -> Result<()> { + let expected = table_schema.as_struct().fields().len(); + if columns != expected { + return plan_err!( + "PartitionExpr needs one child per table column: expected {expected}, got \ + {columns}" + ); + } + Ok(()) +} + +// Equal when the spec, schema and columns are equal. The calculator is derived from +// the first two, so it takes no part. Both comparisons +// include `spec_id` and `schema_id`, which `Hash` relies on. impl PartialEq for PartitionExpr { fn eq(&self, other: &Self) -> bool { self.partition_spec == other.partition_spec && self.table_schema == other.table_schema + && self.columns == other.columns } } @@ -200,44 +272,55 @@ impl PhysicalExpr for PartitionExpr { } fn evaluate(&self, batch: &RecordBatch) -> Result { + // The calculator reads sources by their position in the table, so give it + // the columns laid out as the table. + let columns = self + .columns + .iter() + .map(|column| column.evaluate(batch)?.into_array(batch.num_rows())) + .collect::>>()?; + let fields: Vec = self + .table_schema + .as_struct() + .fields() + .iter() + .zip(&columns) + .map(|(field, column)| { + Field::new(&field.name, column.data_type().clone(), true) + }) + .collect(); + let table_batch = + RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns)?; let array = self .calculator - .calculate(batch) + .calculate(&table_batch) .map_err(to_datafusion_error)?; Ok(ColumnarValue::Array(array)) } fn children(&self) -> Vec<&Arc> { - vec![] + self.columns.iter().collect() } fn with_new_children( self: Arc, - _children: Vec>, + children: Vec>, ) -> Result> { - Ok(self) + check_column_count(&self.table_schema, children.len())?; + Ok(Arc::new(Self { + columns: children, + ..self.as_ref().clone() + })) } fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let field_names: Vec = self - .partition_spec - .fields() - .iter() - .map(|pf| format!("{}({})", pf.transform, pf.name)) - .collect(); - write!(f, "iceberg_partition_values[{}]", field_names.join(", ")) + self.fmt_with(f, |column| fmt_sql(column).to_string()) } } impl std::fmt::Display for PartitionExpr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let field_names: Vec<&str> = self - .partition_spec - .fields() - .iter() - .map(|pf| pf.name.as_str()) - .collect(); - write!(f, "iceberg_partition_values({})", field_names.join(", ")) + self.fmt_with(f, |column| column.to_string()) } } @@ -247,6 +330,7 @@ impl std::hash::Hash for PartitionExpr { // equal expressions agree on both, and collisions fall through to eq. self.partition_spec.spec_id().hash(state); self.table_schema.schema_id().hash(state); + self.columns.hash(state); } } @@ -257,6 +341,7 @@ mod tests { use datafusion::arrow::array::{ArrayRef, Int32Array, StructArray}; use datafusion::arrow::datatypes::{DataType, Field, Fields}; + use datafusion::arrow::util::pretty::pretty_format_columns; use datafusion::physical_plan::empty::EmptyExec; use iceberg::Runtime; use iceberg::spec::{ @@ -271,6 +356,20 @@ mod tests { hasher.finish() } + /// One column per top-level field of `schema`, in table order, as + /// `project_with_partition` passes them. + fn table_columns(schema: &IcebergSchema) -> Vec> { + schema + .as_struct() + .fields() + .iter() + .enumerate() + .map(|(index, field)| { + Arc::new(Column::new(&field.name, index)) as Arc + }) + .collect() + } + // `part` typed per caller, so a spec built against one schema can be paired with // a schema that no longer matches it, as happens when a codec rebuilds. fn schema_with_part(part_type: PrimitiveType) -> IcebergSchemaRef { @@ -350,8 +449,12 @@ mod tests { } let partition_expr = Arc::new( - PartitionExpr::try_new(partition_spec, Arc::new(table_schema.clone())) - .unwrap(), + PartitionExpr::try_new( + partition_spec, + Arc::new(table_schema.clone()), + table_columns(&table_schema), + ) + .unwrap(), ); projection_exprs .push((partition_expr, PROJECTED_PARTITION_VALUE_COLUMN.to_string())); @@ -404,8 +507,12 @@ mod tests { let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); let partition_type = calculator.partition_arrow_type().clone(); - let expr = PartitionExpr::try_new(partition_spec, Arc::new(table_schema.clone())) - .unwrap(); + let expr = PartitionExpr::try_new( + partition_spec, + Arc::new(table_schema.clone()), + table_columns(&table_schema), + ) + .unwrap(); assert_eq!(expr.data_type(&arrow_schema).unwrap(), partition_type); assert!(!expr.nullable(&arrow_schema).unwrap()); @@ -458,13 +565,15 @@ mod tests { ) .unwrap(); - let expr = PartitionExpr::try_new(partition_spec, table_schema).unwrap(); + let columns = table_columns(&table_schema); + let expr = PartitionExpr::try_new(partition_spec, table_schema, columns).unwrap(); // Rebuild from deep copies, as a codec would; cloning the Arcs instead // would let a pointer-based impl pass. let rebuilt = PartitionExpr::try_new( Arc::new(expr.partition_spec().as_ref().clone()), Arc::new(expr.table_schema().as_ref().clone()), + vec![Arc::new(Column::new("id", 0))], ) .unwrap(); @@ -503,33 +612,55 @@ mod tests { .unwrap(), ); - // try_new takes the spec and schema separately, so a codec can pair two that - // do not fit. project_with_partition reaches none of these: it returns early - // when unpartitioned and reads both from the same table metadata. + // try_new takes the spec, schema and columns separately, so a codec can pass + // ones that do not fit. project_with_partition reaches none of these: it + // returns early when unpartitioned and derives all three from the table. let cases = [ ( Arc::new(PartitionSpec::builder(int_schema.clone()).build().unwrap()), int_schema.clone(), - "unpartitioned", + table_columns(&int_schema), ), ( on_part(Transform::Identity), - without_part, - "Field not found", + without_part.clone(), + table_columns(&without_part), ), ( on_part(Transform::Bucket(4)), schema_with_part(PrimitiveType::Boolean), - "not a valid input type of bucket transform", + table_columns(&int_schema), + ), + ( + on_part(Transform::Identity), + int_schema.clone(), + table_columns(&without_part), ), ]; - for (spec, schema, expected) in cases { - let err = PartitionExpr::try_new(spec, schema) - .unwrap_err() - .to_string(); - assert!(err.contains(expected), "expected {expected:?}, got {err}"); - } + let errors: Vec = cases + .into_iter() + .map(|(spec, schema, columns)| { + PartitionExpr::try_new(spec, schema, columns) + .unwrap_err() + .strip_backtrace() + }) + .collect(); + insta::assert_snapshot!(errors.join("\n\n"), @r" + IcebergError(DataInvalid) + caused by + Execution error: DataInvalid => Cannot create partition calculator for unpartitioned table + + IcebergError(Unexpected) + caused by + Execution error: Unexpected, context: { field_id: 2 } => Field not found + + IcebergError(DataInvalid) + caused by + Execution error: DataInvalid => boolean is not a valid input type of bucket transform + + Error during planning: PartitionExpr needs one child per table column: expected 2, got 1 + "); } #[test] @@ -556,7 +687,14 @@ mod tests { .unwrap(), ) }; - let expr = |spec| PartitionExpr::try_new(spec, table_schema.clone()).unwrap(); + let expr = |spec| { + PartitionExpr::try_new( + spec, + table_schema.clone(), + table_columns(&table_schema), + ) + .unwrap() + }; // The same inputs describe the same expression. assert_eq!(expr(spec(1, "id")), expr(spec(1, "id"))); @@ -585,8 +723,18 @@ mod tests { .unwrap(), ); - let on_int = PartitionExpr::try_new(shared.clone(), int_schema).unwrap(); - let on_long = PartitionExpr::try_new(shared, long_schema).unwrap(); + let on_int = PartitionExpr::try_new( + shared.clone(), + int_schema.clone(), + table_columns(&int_schema), + ) + .unwrap(); + let on_long = PartitionExpr::try_new( + shared, + long_schema.clone(), + table_columns(&long_schema), + ) + .unwrap(); assert_eq!( on_int.table_schema().schema_id(), @@ -595,6 +743,76 @@ mod tests { assert_ne!(on_int, on_long); } + #[test] + fn test_partition_expr_reads_columns_through_its_children() { + let table_schema = schema_with_part(PrimitiveType::Int); + let spec = Arc::new( + PartitionSpec::builder(table_schema.clone()) + .add_partition_field("part", "p", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let columns = table_columns(&table_schema); + let expr: Arc = + Arc::new(PartitionExpr::try_new(spec, table_schema, columns).unwrap()); + assert_eq!( + expr.to_string(), + "iceberg_partition_values(p: identity(part@1))" + ); + + // An input laid out unlike the table, as when the optimizer merges the + // partition projection into one that reorders columns. Rewriting the + // children is all it takes to read `part` from its new position. + let moved = Arc::clone(&expr) + .with_new_children(vec![ + Arc::new(Column::new("id", 1)), + Arc::new(Column::new("part", 0)), + ]) + .unwrap(); + assert_eq!( + moved.to_string(), + "iceberg_partition_values(p: identity(part@0))" + ); + let batch = RecordBatch::try_from_iter([ + ("part", Arc::new(Int32Array::from(vec![7, 8])) as ArrayRef), + ("id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef), + ]) + .unwrap(); + let values = moved.evaluate(&batch).unwrap().into_array(2).unwrap(); + insta::assert_snapshot!(pretty_format_columns("_partition", &[values]).unwrap(), @r" + +------------+ + | _partition | + +------------+ + | {p: 7} | + | {p: 8} | + +------------+ + "); + + // The children are part of the expression, so DataFusion can't reuse one + // in place of the other. + assert_ne!(*expr, *moved); + } + + #[test] + fn test_with_new_children_requires_one_child_per_table_column() { + let table_schema = schema_with_part(PrimitiveType::Int); + let spec = Arc::new( + PartitionSpec::builder(table_schema.clone()) + .add_partition_field("part", "p", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let columns = table_columns(&table_schema); + let expr = Arc::new(PartitionExpr::try_new(spec, table_schema, columns).unwrap()); + + let err = expr + .with_new_children(vec![Arc::new(Column::new("part", 0))]) + .unwrap_err(); + insta::assert_snapshot!(err.strip_backtrace(), @"Error during planning: PartitionExpr needs one child per table column: expected 2, got 1"); + } + #[test] fn test_nested_partition() { let address_struct = StructType::new(vec![ @@ -659,9 +877,19 @@ mod tests { ) .unwrap(); - let calculator = - PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - let array = calculator.calculate(&batch).unwrap(); + let columns = table_columns(&table_schema); + let expr = PartitionExpr::try_new( + Arc::new(partition_spec), + Arc::new(table_schema), + columns, + ) + .unwrap(); + // The child shown is the top-level column holding the nested source. + assert_eq!( + expr.to_string(), + "iceberg_partition_values(city_partition: identity(address@1))" + ); + let array = expr.evaluate(&batch).unwrap().into_array(2).unwrap(); let struct_array = array.as_any().downcast_ref::().unwrap(); let city_partition = struct_array diff --git a/crates/datafusion/tests/insert_into.rs b/crates/datafusion/tests/insert_into.rs index 58ca01d..15785db 100644 --- a/crates/datafusion/tests/insert_into.rs +++ b/crates/datafusion/tests/insert_into.rs @@ -31,7 +31,7 @@ use iceberg::spec::{ Literal, NestedField, PrimitiveType, Schema, Struct, StructType, Transform, Type, }; use iceberg_datafusion::IcebergCatalogProvider; -use iceberg_datafusion_test_tools::{FixtureCatalog, display_schema}; +use iceberg_datafusion_test_tools::{FixtureCatalog, IcebergTestHarness, display_schema}; #[tokio::test] async fn test_insert_into() -> Result<(), Box> { @@ -489,3 +489,74 @@ async fn test_insert_into_partitioned() -> Result<(), Box> { Ok(()) } + +/// An INSERT whose input ends in a projection that renames and reorders columns gets +/// the partition projection merged into it by the optimizer. The partition values +/// must still come from the columns being written (#35). +#[tokio::test] +async fn test_insert_into_partitioned_through_reordering_projection() +-> Result<(), Box> { + let fixtures = FixtureCatalog::try_new().await?; + fixtures + .table( + TableIdent::from_strs(["default", "t"])?, + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)) + .into(), + NestedField::required(2, "val", Type::Primitive(PrimitiveType::Int)) + .into(), + ]) + .build()?, + ) + .partition_by("id", Transform::Identity) + .create() + .await?; + let harness = IcebergTestHarness::builder() + .with_catalog( + "iceberg", + Arc::new(IcebergCatalogProvider::try_new(fixtures.catalog()).await?), + ) + // One partition keeps repartitioning out of the plan. + .with_config_option("datafusion.execution.target_partitions", 1) + .build() + .await?; + + // `id` is written from `column2`, the second source column, so the partition + // values must read `column2@1`, not the first column of the merged projection's + // input. + let (plan, count) = harness + .query( + "INSERT INTO iceberg.default.t + SELECT column2 AS id, column1 AS val FROM (VALUES (1, 100), (2, 200))", + ) + .await?; + insta::assert_snapshot!(plan, @r#" + IcebergCommitExec: table=default.t, schema=[1: id: required int, 2: val: required int] + IcebergWriteExec: table=default.t, result_schema=Schema { fields: [Field { name: "data_files", data_type: LargeBinary }], metadata: {} } + ProjectionExec: expr=[column2@1 as id, column1@0 as val, iceberg_partition_values(id: identity(column2@1)) as _partition] + DataSourceExec: partitions=1, partition_sizes=[1] + "#); + insta::assert_snapshot!(count, @r" + +-------+ + | count | + +-------+ + | 2 | + +-------+ + "); + + // A filter on the partition column prunes files by their partition values, so it + // only finds the row if they're right. + let (_, rows) = harness + .query("SELECT * FROM iceberg.default.t WHERE id = 100") + .await?; + insta::assert_snapshot!(rows, @r" + +-----+-----+ + | id | val | + +-----+-----+ + | 100 | 1 | + +-----+-----+ + "); + + Ok(()) +}