Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,6 @@ impl TableProvider for IcebergCatalogTableProvider {
let repartitioned_plan =
repartition(plan_with_partition, table.metadata_ref(), target_partitions)?;

// Apply sort node when it's not fanout mode
let fanout_enabled = table
.metadata()
.properties()
Expand All @@ -223,7 +222,10 @@ impl TableProvider for IcebergCatalogTableProvider {
.transpose()?
.unwrap_or(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT);

let write_input = if fanout_enabled {
// Without fanout, a partitioned table is written by the clustered writer,
// which needs its input sorted by partition. Unpartitioned tables have no
// partition column to sort by.
let write_input = if fanout_enabled || partition_spec.is_unpartitioned() {
repartitioned_plan
} else {
sort_by_partition(repartitioned_plan)?
Expand Down
65 changes: 64 additions & 1 deletion crates/datafusion/tests/insert_into.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ use iceberg::spec::{
Literal, NestedField, PrimitiveType, Schema, Struct, StructType, Transform, Type,
};
use iceberg_datafusion::IcebergCatalogProvider;
use iceberg_datafusion_test_tools::{FixtureCatalog, display_schema};
use iceberg_datafusion_test_tools::{FixtureCatalog, IcebergTestHarness, display_schema};

#[tokio::test]
async fn test_insert_into() -> Result<(), Box<dyn Error>> {
Expand Down Expand Up @@ -489,3 +489,66 @@ async fn test_insert_into_partitioned() -> Result<(), Box<dyn Error>> {

Ok(())
}

/// With fanout disabled, only partitioned tables are sorted before writing:
/// an unpartitioned table has no partition column to sort by.
#[tokio::test]
async fn inserts_into_an_unpartitioned_table_without_fanout() -> Result<(), Box<dyn Error>>
{
let fixtures = FixtureCatalog::try_new().await?;
fixtures
.table(
TableIdent::from_strs(["default", "t"])?,
Schema::builder()
.with_fields(vec![
NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int))
.into(),
NestedField::required(2, "val", Type::Primitive(PrimitiveType::Int))
.into(),
])
.build()?,
)
.property("write.datafusion.fanout.enabled", "false")
.create()
.await?;
let harness = IcebergTestHarness::builder()
.with_catalog(
"iceberg",
Arc::new(IcebergCatalogProvider::try_new(fixtures.catalog()).await?),
)
// One partition keeps repartitioning out of the plan.
.with_config_option("datafusion.execution.target_partitions", 1)
.build()
.await?;

let (plan, count) = harness
.query("INSERT INTO iceberg.default.t VALUES (1, 10), (2, 20)")
.await?;
insta::assert_snapshot!(plan, @r#"
IcebergCommitExec: table=default.t, schema=[1: id: required int, 2: val: required int]
IcebergWriteExec: table=default.t, result_schema=Schema { fields: [Field { name: "data_files", data_type: LargeBinary }], metadata: {} }
ProjectionExec: expr=[column1@0 as id, column2@1 as val]
DataSourceExec: partitions=1, partition_sizes=[1]
"#);
insta::assert_snapshot!(count, @r"
+-------+
| count |
+-------+
| 2 |
+-------+
");

let (_, rows) = harness
.query("SELECT * FROM iceberg.default.t ORDER BY id")
.await?;
insta::assert_snapshot!(rows, @r"
+----+-----+
| id | val |
+----+-----+
| 1 | 10 |
| 2 | 20 |
+----+-----+
");

Ok(())
}