diff --git a/AGENTS.md b/AGENTS.md index 1859768..c6df87b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,7 +108,7 @@ input - **`TaskWriter`** picks one of three writers: `UnpartitionedWriter`; `FanoutWriter` (accepts unsorted input, keeps many files open); or `ClusteredWriter` (requires sorted input, which is why the sort step exists). - **Writer limits:** only Parquet is supported. The writer honors `write.parquet.*` table properties and table encryption. It matches columns by name, because DataFusion batches carry no field IDs. -- **`PartitionExpr`** is public and retains its partition spec and table schema so that distributed engines can serialize them and rebuild the expression on workers with `try_new`. Its equality and hash are by value; don't go back to pointer equality. +- **`PartitionExpr`** is public and retains its partition spec and table schema so that distributed engines can serialize them and rebuild the expression on workers with `try_new`, passing the decoded children. Its children are the table's top-level columns, in table order, and it reads the batch only through them, so optimizer rules like `ProjectionPushdown` that move it onto a different input rewrite them like any other column reference. Its equality and hash are by value; don't go back to pointer equality. ### Errors diff --git a/crates/datafusion/src/write/project.rs b/crates/datafusion/src/write/project.rs index c62e4d5..c162f12 100644 --- a/crates/datafusion/src/write/project.rs +++ b/crates/datafusion/src/write/project.rs @@ -20,18 +20,22 @@ use std::sync::Arc; use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema}; +use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; use datafusion::common::plan_err; use datafusion::error::Result; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_expr::expressions::Column; +use datafusion::physical_expr_common::physical_expr::fmt_sql; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::{ColumnarValue, ExecutionPlan}; use iceberg::arrow::{ PROJECTED_PARTITION_VALUE_COLUMN, PartitionValueCalculator, schema_to_arrow_schema, strip_metadata_from_schema, }; -use iceberg::spec::{PartitionSpec, SchemaRef as IcebergSchemaRef}; +use iceberg::spec::{ + NestedField, PartitionSpec, Schema as IcebergSchema, SchemaRef as IcebergSchemaRef, + Type, +}; use iceberg::table::Table; use crate::to_datafusion_error; @@ -91,6 +95,10 @@ pub fn project_with_partition( let partition_expr = Arc::new(PartitionExpr::try_new( partition_spec.clone(), table_schema.clone(), + projection_exprs + .iter() + .map(|(column, _)| Arc::clone(column)) + .collect(), )?); projection_exprs.push((partition_expr, PROJECTED_PARTITION_VALUE_COLUMN.to_string())); @@ -100,30 +108,41 @@ pub fn project_with_partition( /// PhysicalExpr implementation for partition value calculation /// +/// Its children are the table's top-level columns, in table order, and it reads the +/// batch only through them. Optimizer rules that move it onto a different input, +/// such as merging it into the projection below, rewrite them like any other column +/// reference. +/// /// The [`PartitionValueCalculator`] cannot be serialized, so the spec and schema it -/// was built from are retained for [`Self::try_new`] to rebuild from. +/// was built from are retained for [`Self::try_new`] to rebuild from, with the +/// decoded children. #[derive(Debug, Clone)] pub struct PartitionExpr { calculator: Arc, partition_spec: Arc, table_schema: IcebergSchemaRef, + columns: Vec>, } impl PartitionExpr { - /// Builds the expression from the spec and schema that define it. + /// Builds the expression from the spec and schema that define it, reading the + /// table's top-level columns from `columns`, one per column in table order. /// /// The calculator is built here rather than passed in, so it cannot drift from /// the retained inputs. /// /// # Errors /// - /// Returns an error if the spec is unpartitioned or cannot be bound to the schema. + /// Returns an error if the spec is unpartitioned or cannot be bound to the schema, + /// or if `columns` doesn't have one expression per top-level table column. /// /// # Example /// /// ``` /// use std::sync::Arc; /// + /// use datafusion::physical_expr::PhysicalExpr; + /// use datafusion::physical_expr::expressions::Column; /// use iceberg_datafusion::write::PartitionExpr; /// use iceberg::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, Transform, Type}; /// @@ -139,12 +158,14 @@ impl PartitionExpr { /// .add_partition_field("id", "id_bucket", Transform::Bucket(16))? /// .build()?, /// ); - /// let expr = PartitionExpr::try_new(spec, schema)?; + /// let expr = PartitionExpr::try_new(spec, schema, vec![Arc::new(Column::new("id", 0))])?; /// - /// // A worker rebuilds an equal expression from the two retained inputs. + /// // A worker rebuilds an equal expression from the retained spec and schema and + /// // the decoded children. /// let rebuilt = PartitionExpr::try_new( /// Arc::new(expr.partition_spec().as_ref().clone()), /// Arc::new(expr.table_schema().as_ref().clone()), + /// expr.children().into_iter().cloned().collect(), /// )?; /// assert_eq!(expr, rebuilt); /// # Ok::<(), Box>(()) @@ -152,21 +173,25 @@ impl PartitionExpr { pub fn try_new( partition_spec: Arc, table_schema: IcebergSchemaRef, + columns: Vec>, ) -> Result { let calculator = PartitionValueCalculator::try_new( partition_spec.as_ref(), table_schema.as_ref(), ) .map_err(to_datafusion_error)?; + check_column_count(&table_schema, columns.len())?; Ok(Self { calculator: Arc::new(calculator), partition_spec, table_schema, + columns, }) } /// The partition spec this expression computes values for. With - /// [`Self::table_schema`], all [`Self::try_new`] needs to rebuild an equal one. + /// [`Self::table_schema`] and the children, all [`Self::try_new`] needs to + /// rebuild an equal one. pub fn partition_spec(&self) -> &Arc { &self.partition_spec } @@ -176,15 +201,62 @@ impl PartitionExpr { pub fn table_schema(&self) -> &IcebergSchemaRef { &self.table_schema } + + /// Writes each partition field with the column holding its source, e.g. + /// `iceberg_partition_values(id: identity(b@1))`. + fn fmt_with( + &self, + f: &mut std::fmt::Formatter<'_>, + fmt_column: impl Fn(&dyn PhysicalExpr) -> String, + ) -> std::fmt::Result { + let table_columns = self.table_schema.as_struct().fields(); + let fields: Vec = self + .partition_spec + .fields() + .iter() + .map(|field| { + // `try_new` checked that the source is in the schema. + let column = table_columns + .iter() + .position(|column| holds(column, field.source_id)) + .map(|index| fmt_column(self.columns[index].as_ref())) + .unwrap_or_default(); + format!("{}: {}({column})", field.name, field.transform) + }) + .collect(); + write!(f, "iceberg_partition_values({})", fields.join(", ")) + } } -// Equal when the spec and schema are equal. The calculator is derived from them, so -// it takes no part. Both comparisons include `spec_id` and `schema_id`, which `Hash` -// relies on. +/// Whether `field` is the field with `id` or a struct containing it. Partition +/// sources can't be nested in lists or maps. +fn holds(field: &NestedField, id: i32) -> bool { + field.id == id + || matches!( + field.field_type.as_ref(), + Type::Struct(fields) if fields.fields().iter().any(|field| holds(field, id)) + ) +} + +fn check_column_count(table_schema: &IcebergSchema, columns: usize) -> Result<()> { + let expected = table_schema.as_struct().fields().len(); + if columns != expected { + return plan_err!( + "PartitionExpr needs one child per table column: expected {expected}, got \ + {columns}" + ); + } + Ok(()) +} + +// Equal when the spec, schema and columns are equal. The calculator is derived from +// the first two, so it takes no part. Both comparisons +// include `spec_id` and `schema_id`, which `Hash` relies on. impl PartialEq for PartitionExpr { fn eq(&self, other: &Self) -> bool { self.partition_spec == other.partition_spec && self.table_schema == other.table_schema + && self.columns == other.columns } } @@ -200,44 +272,55 @@ impl PhysicalExpr for PartitionExpr { } fn evaluate(&self, batch: &RecordBatch) -> Result { + // The calculator reads sources by their position in the table, so give it + // the columns laid out as the table. + let columns = self + .columns + .iter() + .map(|column| column.evaluate(batch)?.into_array(batch.num_rows())) + .collect::>>()?; + let fields: Vec = self + .table_schema + .as_struct() + .fields() + .iter() + .zip(&columns) + .map(|(field, column)| { + Field::new(&field.name, column.data_type().clone(), true) + }) + .collect(); + let table_batch = + RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns)?; let array = self .calculator - .calculate(batch) + .calculate(&table_batch) .map_err(to_datafusion_error)?; Ok(ColumnarValue::Array(array)) } fn children(&self) -> Vec<&Arc> { - vec![] + self.columns.iter().collect() } fn with_new_children( self: Arc, - _children: Vec>, + children: Vec>, ) -> Result> { - Ok(self) + check_column_count(&self.table_schema, children.len())?; + Ok(Arc::new(Self { + columns: children, + ..self.as_ref().clone() + })) } fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let field_names: Vec = self - .partition_spec - .fields() - .iter() - .map(|pf| format!("{}({})", pf.transform, pf.name)) - .collect(); - write!(f, "iceberg_partition_values[{}]", field_names.join(", ")) + self.fmt_with(f, |column| fmt_sql(column).to_string()) } } impl std::fmt::Display for PartitionExpr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let field_names: Vec<&str> = self - .partition_spec - .fields() - .iter() - .map(|pf| pf.name.as_str()) - .collect(); - write!(f, "iceberg_partition_values({})", field_names.join(", ")) + self.fmt_with(f, |column| column.to_string()) } } @@ -247,6 +330,7 @@ impl std::hash::Hash for PartitionExpr { // equal expressions agree on both, and collisions fall through to eq. self.partition_spec.spec_id().hash(state); self.table_schema.schema_id().hash(state); + self.columns.hash(state); } } @@ -257,6 +341,7 @@ mod tests { use datafusion::arrow::array::{ArrayRef, Int32Array, StructArray}; use datafusion::arrow::datatypes::{DataType, Field, Fields}; + use datafusion::arrow::util::pretty::pretty_format_columns; use datafusion::physical_plan::empty::EmptyExec; use iceberg::Runtime; use iceberg::spec::{ @@ -271,6 +356,20 @@ mod tests { hasher.finish() } + /// One column per top-level field of `schema`, in table order, as + /// `project_with_partition` passes them. + fn table_columns(schema: &IcebergSchema) -> Vec> { + schema + .as_struct() + .fields() + .iter() + .enumerate() + .map(|(index, field)| { + Arc::new(Column::new(&field.name, index)) as Arc + }) + .collect() + } + // `part` typed per caller, so a spec built against one schema can be paired with // a schema that no longer matches it, as happens when a codec rebuilds. fn schema_with_part(part_type: PrimitiveType) -> IcebergSchemaRef { @@ -350,8 +449,12 @@ mod tests { } let partition_expr = Arc::new( - PartitionExpr::try_new(partition_spec, Arc::new(table_schema.clone())) - .unwrap(), + PartitionExpr::try_new( + partition_spec, + Arc::new(table_schema.clone()), + table_columns(&table_schema), + ) + .unwrap(), ); projection_exprs .push((partition_expr, PROJECTED_PARTITION_VALUE_COLUMN.to_string())); @@ -404,8 +507,12 @@ mod tests { let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); let partition_type = calculator.partition_arrow_type().clone(); - let expr = PartitionExpr::try_new(partition_spec, Arc::new(table_schema.clone())) - .unwrap(); + let expr = PartitionExpr::try_new( + partition_spec, + Arc::new(table_schema.clone()), + table_columns(&table_schema), + ) + .unwrap(); assert_eq!(expr.data_type(&arrow_schema).unwrap(), partition_type); assert!(!expr.nullable(&arrow_schema).unwrap()); @@ -458,13 +565,15 @@ mod tests { ) .unwrap(); - let expr = PartitionExpr::try_new(partition_spec, table_schema).unwrap(); + let columns = table_columns(&table_schema); + let expr = PartitionExpr::try_new(partition_spec, table_schema, columns).unwrap(); // Rebuild from deep copies, as a codec would; cloning the Arcs instead // would let a pointer-based impl pass. let rebuilt = PartitionExpr::try_new( Arc::new(expr.partition_spec().as_ref().clone()), Arc::new(expr.table_schema().as_ref().clone()), + vec![Arc::new(Column::new("id", 0))], ) .unwrap(); @@ -503,33 +612,55 @@ mod tests { .unwrap(), ); - // try_new takes the spec and schema separately, so a codec can pair two that - // do not fit. project_with_partition reaches none of these: it returns early - // when unpartitioned and reads both from the same table metadata. + // try_new takes the spec, schema and columns separately, so a codec can pass + // ones that do not fit. project_with_partition reaches none of these: it + // returns early when unpartitioned and derives all three from the table. let cases = [ ( Arc::new(PartitionSpec::builder(int_schema.clone()).build().unwrap()), int_schema.clone(), - "unpartitioned", + table_columns(&int_schema), ), ( on_part(Transform::Identity), - without_part, - "Field not found", + without_part.clone(), + table_columns(&without_part), ), ( on_part(Transform::Bucket(4)), schema_with_part(PrimitiveType::Boolean), - "not a valid input type of bucket transform", + table_columns(&int_schema), + ), + ( + on_part(Transform::Identity), + int_schema.clone(), + table_columns(&without_part), ), ]; - for (spec, schema, expected) in cases { - let err = PartitionExpr::try_new(spec, schema) - .unwrap_err() - .to_string(); - assert!(err.contains(expected), "expected {expected:?}, got {err}"); - } + let errors: Vec = cases + .into_iter() + .map(|(spec, schema, columns)| { + PartitionExpr::try_new(spec, schema, columns) + .unwrap_err() + .strip_backtrace() + }) + .collect(); + insta::assert_snapshot!(errors.join("\n\n"), @r" + IcebergError(DataInvalid) + caused by + Execution error: DataInvalid => Cannot create partition calculator for unpartitioned table + + IcebergError(Unexpected) + caused by + Execution error: Unexpected, context: { field_id: 2 } => Field not found + + IcebergError(DataInvalid) + caused by + Execution error: DataInvalid => boolean is not a valid input type of bucket transform + + Error during planning: PartitionExpr needs one child per table column: expected 2, got 1 + "); } #[test] @@ -556,7 +687,14 @@ mod tests { .unwrap(), ) }; - let expr = |spec| PartitionExpr::try_new(spec, table_schema.clone()).unwrap(); + let expr = |spec| { + PartitionExpr::try_new( + spec, + table_schema.clone(), + table_columns(&table_schema), + ) + .unwrap() + }; // The same inputs describe the same expression. assert_eq!(expr(spec(1, "id")), expr(spec(1, "id"))); @@ -585,8 +723,18 @@ mod tests { .unwrap(), ); - let on_int = PartitionExpr::try_new(shared.clone(), int_schema).unwrap(); - let on_long = PartitionExpr::try_new(shared, long_schema).unwrap(); + let on_int = PartitionExpr::try_new( + shared.clone(), + int_schema.clone(), + table_columns(&int_schema), + ) + .unwrap(); + let on_long = PartitionExpr::try_new( + shared, + long_schema.clone(), + table_columns(&long_schema), + ) + .unwrap(); assert_eq!( on_int.table_schema().schema_id(), @@ -595,6 +743,76 @@ mod tests { assert_ne!(on_int, on_long); } + #[test] + fn test_partition_expr_reads_columns_through_its_children() { + let table_schema = schema_with_part(PrimitiveType::Int); + let spec = Arc::new( + PartitionSpec::builder(table_schema.clone()) + .add_partition_field("part", "p", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let columns = table_columns(&table_schema); + let expr: Arc = + Arc::new(PartitionExpr::try_new(spec, table_schema, columns).unwrap()); + assert_eq!( + expr.to_string(), + "iceberg_partition_values(p: identity(part@1))" + ); + + // An input laid out unlike the table, as when the optimizer merges the + // partition projection into one that reorders columns. Rewriting the + // children is all it takes to read `part` from its new position. + let moved = Arc::clone(&expr) + .with_new_children(vec![ + Arc::new(Column::new("id", 1)), + Arc::new(Column::new("part", 0)), + ]) + .unwrap(); + assert_eq!( + moved.to_string(), + "iceberg_partition_values(p: identity(part@0))" + ); + let batch = RecordBatch::try_from_iter([ + ("part", Arc::new(Int32Array::from(vec![7, 8])) as ArrayRef), + ("id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef), + ]) + .unwrap(); + let values = moved.evaluate(&batch).unwrap().into_array(2).unwrap(); + insta::assert_snapshot!(pretty_format_columns("_partition", &[values]).unwrap(), @r" + +------------+ + | _partition | + +------------+ + | {p: 7} | + | {p: 8} | + +------------+ + "); + + // The children are part of the expression, so DataFusion can't reuse one + // in place of the other. + assert_ne!(*expr, *moved); + } + + #[test] + fn test_with_new_children_requires_one_child_per_table_column() { + let table_schema = schema_with_part(PrimitiveType::Int); + let spec = Arc::new( + PartitionSpec::builder(table_schema.clone()) + .add_partition_field("part", "p", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let columns = table_columns(&table_schema); + let expr = Arc::new(PartitionExpr::try_new(spec, table_schema, columns).unwrap()); + + let err = expr + .with_new_children(vec![Arc::new(Column::new("part", 0))]) + .unwrap_err(); + insta::assert_snapshot!(err.strip_backtrace(), @"Error during planning: PartitionExpr needs one child per table column: expected 2, got 1"); + } + #[test] fn test_nested_partition() { let address_struct = StructType::new(vec![ @@ -659,9 +877,19 @@ mod tests { ) .unwrap(); - let calculator = - PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - let array = calculator.calculate(&batch).unwrap(); + let columns = table_columns(&table_schema); + let expr = PartitionExpr::try_new( + Arc::new(partition_spec), + Arc::new(table_schema), + columns, + ) + .unwrap(); + // The child shown is the top-level column holding the nested source. + assert_eq!( + expr.to_string(), + "iceberg_partition_values(city_partition: identity(address@1))" + ); + let array = expr.evaluate(&batch).unwrap().into_array(2).unwrap(); let struct_array = array.as_any().downcast_ref::().unwrap(); let city_partition = struct_array diff --git a/crates/datafusion/tests/insert_into.rs b/crates/datafusion/tests/insert_into.rs index 58ca01d..15785db 100644 --- a/crates/datafusion/tests/insert_into.rs +++ b/crates/datafusion/tests/insert_into.rs @@ -31,7 +31,7 @@ use iceberg::spec::{ Literal, NestedField, PrimitiveType, Schema, Struct, StructType, Transform, Type, }; use iceberg_datafusion::IcebergCatalogProvider; -use iceberg_datafusion_test_tools::{FixtureCatalog, display_schema}; +use iceberg_datafusion_test_tools::{FixtureCatalog, IcebergTestHarness, display_schema}; #[tokio::test] async fn test_insert_into() -> Result<(), Box> { @@ -489,3 +489,74 @@ async fn test_insert_into_partitioned() -> Result<(), Box> { Ok(()) } + +/// An INSERT whose input ends in a projection that renames and reorders columns gets +/// the partition projection merged into it by the optimizer. The partition values +/// must still come from the columns being written (#35). +#[tokio::test] +async fn test_insert_into_partitioned_through_reordering_projection() +-> Result<(), Box> { + let fixtures = FixtureCatalog::try_new().await?; + fixtures + .table( + TableIdent::from_strs(["default", "t"])?, + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)) + .into(), + NestedField::required(2, "val", Type::Primitive(PrimitiveType::Int)) + .into(), + ]) + .build()?, + ) + .partition_by("id", Transform::Identity) + .create() + .await?; + let harness = IcebergTestHarness::builder() + .with_catalog( + "iceberg", + Arc::new(IcebergCatalogProvider::try_new(fixtures.catalog()).await?), + ) + // One partition keeps repartitioning out of the plan. + .with_config_option("datafusion.execution.target_partitions", 1) + .build() + .await?; + + // `id` is written from `column2`, the second source column, so the partition + // values must read `column2@1`, not the first column of the merged projection's + // input. + let (plan, count) = harness + .query( + "INSERT INTO iceberg.default.t + SELECT column2 AS id, column1 AS val FROM (VALUES (1, 100), (2, 200))", + ) + .await?; + insta::assert_snapshot!(plan, @r#" + IcebergCommitExec: table=default.t, schema=[1: id: required int, 2: val: required int] + IcebergWriteExec: table=default.t, result_schema=Schema { fields: [Field { name: "data_files", data_type: LargeBinary }], metadata: {} } + ProjectionExec: expr=[column2@1 as id, column1@0 as val, iceberg_partition_values(id: identity(column2@1)) as _partition] + DataSourceExec: partitions=1, partition_sizes=[1] + "#); + insta::assert_snapshot!(count, @r" + +-------+ + | count | + +-------+ + | 2 | + +-------+ + "); + + // A filter on the partition column prunes files by their partition values, so it + // only finds the row if they're right. + let (_, rows) = harness + .query("SELECT * FROM iceberg.default.t WHERE id = 100") + .await?; + insta::assert_snapshot!(rows, @r" + +-----+-----+ + | id | val | + +-----+-----+ + | 100 | 1 | + +-----+-----+ + "); + + Ok(()) +}