diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 6dd6bd5f37..64fe61f17b 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3483,6 +3483,7 @@ pub fn iceberg::table::Table::encryption_manager(&self) -> core::option::Option< pub fn iceberg::table::Table::file_io(&self) -> &iceberg::io::FileIO pub fn iceberg::table::Table::identifier(&self) -> &iceberg::TableIdent pub fn iceberg::table::Table::inspect(&self) -> iceberg::inspect::MetadataTable<'_> +pub fn iceberg::table::Table::location_generator(&self) -> iceberg::Result pub fn iceberg::table::Table::manifest_list_reader(&self, snapshot: &iceberg::spec::SnapshotRef) -> iceberg::spec::ManifestListReader pub fn iceberg::table::Table::manifest_reader(&self) -> iceberg::spec::ManifestReader pub fn iceberg::table::Table::metadata(&self) -> &iceberg::spec::TableMetadata @@ -3691,6 +3692,13 @@ impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::Obje pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +pub struct iceberg::writer::file_writer::location_generator::TableLocationGenerator +impl core::clone::Clone for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::clone(&self) -> iceberg::writer::file_writer::location_generator::TableLocationGenerator +impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String pub trait iceberg::writer::file_writer::location_generator::FileNameGenerator: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static pub fn iceberg::writer::file_writer::location_generator::FileNameGenerator::generate_file_name(&self) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::FileNameGenerator for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator @@ -3701,6 +3709,8 @@ impl iceberg::writer::file_writer::location_generator::LocationGenerator for ice pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String pub mod iceberg::writer::file_writer::rolling_writer pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriter impl iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator diff --git a/crates/iceberg/src/cow_rewrite/writer.rs b/crates/iceberg/src/cow_rewrite/writer.rs index ce235e889b..6d8fdb8a3c 100644 --- a/crates/iceberg/src/cow_rewrite/writer.rs +++ b/crates/iceberg/src/cow_rewrite/writer.rs @@ -27,10 +27,7 @@ use crate::table::Table; use crate::writer::IcebergWriterBuilder; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; use crate::writer::file_writer::ParquetWriterBuilder; -use crate::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator, - ObjectStorageLocationGenerator, -}; +use crate::writer::file_writer::location_generator::DefaultFileNameGenerator; use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; /// Builds a boxed replacement-data-file writer. @@ -44,9 +41,8 @@ use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; /// batch is written, so it is safe to construct one optimistically and only /// write to it once a source file is known to have changed. /// -/// The location layout follows `write.object-storage.enabled`: tables that opt -/// into the object-storage layout get hash-entropy paths from -/// [`ObjectStorageLocationGenerator`], everything else gets the default layout. +/// File locations follow the layout selected by the table properties; see +/// [`Table::location_generator`]. pub(crate) async fn build_replacement_writer( table: &Table, write_schema: SchemaRef, @@ -72,27 +68,12 @@ pub(crate) async fn build_replacement_writer( parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone()); } - if table_props.write_object_storage_enabled()? { - let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?; - build_data_file_writer(table, parquet_builder, location_generator, partition_key).await - } else { - let location_generator = DefaultLocationGenerator::new(table.metadata())?; - build_data_file_writer(table, parquet_builder, location_generator, partition_key).await - } -} - -async fn build_data_file_writer( - table: &Table, - parquet_builder: ParquetWriterBuilder, - location_generator: L, - partition_key: Option, -) -> Result> { + let location_generator = table.location_generator()?; let file_name_generator = DefaultFileNameGenerator::new( format!("cow-rewrite-{}", Uuid::now_v7()), None, DataFileFormat::Parquet, ); - let table_props = table.metadata().table_properties(); let rolling_builder = RollingFileWriterBuilder::new( parquet_builder, table_props.write_target_file_size_bytes()?, diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index f33c246eb7..e426a34e22 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -31,6 +31,7 @@ use crate::scan::TableScanBuilder; use crate::spec::{ ManifestListReader, ManifestReader, SchemaRef, SnapshotRef, TableMetadata, TableMetadataRef, }; +use crate::writer::file_writer::location_generator::TableLocationGenerator; use crate::{Result, TableIdent}; /// Builder to create table scan. @@ -325,6 +326,17 @@ impl Table { pub fn reader_builder(&self) -> ArrowReaderBuilder { ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone()) } + + /// Creates the location generator for the layout selected by the table properties. + /// + /// Java's `write.location-provider.impl` is not supported and is ignored. + /// + /// # Errors + /// + /// Returns an error if a table property that determines the layout has an invalid value. + pub fn location_generator(&self) -> Result { + TableLocationGenerator::new(self.metadata()) + } } /// `StaticTable` is a read-only table struct that can be created from a metadata file or from `TableMetaData` without a catalog. diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 26207d3e1d..4e9aca44e4 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -54,6 +54,11 @@ const ENTROPY_DIR_DEPTH: usize = 3; #[derive(Clone, Debug)] /// `DefaultLocationGenerator` used to generate the data dir location of data file. /// The location is generated based on the table location and the data location in table properties. +/// +/// Paths use `{data location}/{partition path}/{file name}`, omitting the +/// partition path for unpartitioned files. It ignores `write.object-storage.enabled`; use +/// [`Table::location_generator`](crate::table::Table::location_generator) for the layout the +/// table properties select. pub struct DefaultLocationGenerator { data_location: String, } @@ -203,6 +208,49 @@ impl LocationGenerator for ObjectStorageLocationGenerator { } } +/// Generates data file paths using the layout configured in table properties. +/// +/// It uses [`ObjectStorageLocationGenerator`] when +/// [`write.object-storage.enabled`](crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED) +/// is `true`, and [`DefaultLocationGenerator`] otherwise. It is created by +/// [`Table::location_generator`](crate::table::Table::location_generator). +#[derive(Clone, Debug)] +pub struct TableLocationGenerator { + inner: Layout, +} + +#[derive(Clone, Debug)] +enum Layout { + Default(DefaultLocationGenerator), + ObjectStorage(ObjectStorageLocationGenerator), +} + +impl TableLocationGenerator { + /// Creates a generator from table metadata. + pub(crate) fn new(table_metadata: &TableMetadata) -> Result { + let inner = if table_metadata + .table_properties() + .write_object_storage_enabled()? + { + Layout::ObjectStorage(ObjectStorageLocationGenerator::new(table_metadata)?) + } else { + Layout::Default(DefaultLocationGenerator::new(table_metadata)?) + }; + Ok(Self { inner }) + } +} + +impl LocationGenerator for TableLocationGenerator { + fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String { + match &self.inner { + Layout::Default(generator) => generator.generate_location(partition_key, file_name), + Layout::ObjectStorage(generator) => { + generator.generate_location(partition_key, file_name) + } + } + } +} + /// Derive the `{parent}/{name}` context from a table location, mirroring Hadoop's /// `Path.getParent().getName()` / `Path.getName()`. fn path_context(table_location: &str) -> String { @@ -291,15 +339,18 @@ pub(crate) mod test { use std::collections::HashMap; use std::sync::Arc; + use rstest::rstest; use uuid::Uuid; use super::LocationGenerator; + use crate::ErrorKind; use crate::spec::{ FormatVersion, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Schema, Struct, StructType, TableMetadata, TableProperties, Transform, Type, }; use crate::writer::file_writer::location_generator::{ DefaultLocationGenerator, FileNameGenerator, ObjectStorageLocationGenerator, + TableLocationGenerator, }; #[test] @@ -609,6 +660,95 @@ pub(crate) mod test { ); } + #[rstest] + #[case::unset(&[], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::disabled(&[(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, "false")], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::java_location_provider(&[("write.location-provider.impl", "com.example.Provider")], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::enabled(&[(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, "true")], [ + "s3://data.db/table/data/0110/1010/0011/11101000/test.parquet", + "s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet", + ])] + fn test_table_location_follows_properties( + #[case] properties: &[(&str, &str)], + #[case] expected: [&str; 2], + ) { + let metadata = table_metadata_with( + "s3://data.db/table", + properties + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "data#1", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("data#1", "data#1", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition_key = PartitionKey::new( + partition_spec, + schema, + Struct::from_iter([Some(Literal::string("val#1"))]), + ); + let generator = TableLocationGenerator::new(&metadata).unwrap(); + + assert_eq!( + [ + generator.generate_location(None, "test.parquet"), + generator.generate_location(Some(&partition_key), "test.parquet"), + ], + expected + ); + } + + #[test] + fn test_table_location_rejects_invalid_property() { + let metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED.to_string(), + "yes".to_string(), + )]), + ); + let err = TableLocationGenerator::new(&metadata).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_default_location_ignores_object_storage_property() { + let metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED.to_string(), + "true".to_string(), + )]), + ); + let generator = DefaultLocationGenerator::new(&metadata).unwrap(); + + assert_eq!( + generator.generate_location(None, "test.parquet"), + "s3://data.db/table/data/test.parquet" + ); + } + /// Build a minimal `TableMetadata` for location generator tests. fn table_metadata_with(location: &str, properties: HashMap) -> TableMetadata { TableMetadata { diff --git a/crates/iceberg/src/writer/mod.rs b/crates/iceberg/src/writer/mod.rs index ca401097ab..0a3850ee81 100644 --- a/crates/iceberg/src/writer/mod.rs +++ b/crates/iceberg/src/writer/mod.rs @@ -50,9 +50,7 @@ //! use iceberg::transaction::Transaction; //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; -//! use iceberg::writer::file_writer::location_generator::{ -//! DefaultFileNameGenerator, DefaultLocationGenerator, -//! }; +//! use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent}; //! use parquet::file::properties::WriterProperties; @@ -78,7 +76,7 @@ //! let table = catalog //! .load_table(&TableIdent::from_strs(["hello", "world"])?) //! .await?; -//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! let location_generator = table.location_generator()?; //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -124,9 +122,7 @@ //! use iceberg::spec::{DataFile, PartitionKey}; //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; -//! use iceberg::writer::file_writer::location_generator::{ -//! DefaultFileNameGenerator, DefaultLocationGenerator, -//! }; +//! use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent}; //! use parquet::file::properties::WriterProperties; @@ -207,7 +203,7 @@ //! table.metadata().current_schema().clone(), //! Struct::from_iter(vec![Some(Literal::string("Seattle"))]), //! ); -//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! let location_generator = table.location_generator()?; //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -261,9 +257,7 @@ //! # use iceberg::{Catalog, CatalogBuilder, Result, TableIdent}; //! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! # use iceberg::writer::file_writer::ParquetWriterBuilder; -//! # use iceberg::writer::file_writer::location_generator::{ -//! # DefaultFileNameGenerator, DefaultLocationGenerator, -//! # }; +//! # use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator; //! # use parquet::file::properties::WriterProperties; //! # use std::collections::HashMap; //! # #[tokio::main] @@ -272,7 +266,7 @@ //! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) //! # .await?; //! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; -//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let location_generator = table.location_generator()?; //! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); //! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); //! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( @@ -325,9 +319,7 @@ //! # use iceberg::{Catalog, CatalogBuilder, Result, TableIdent}; //! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! # use iceberg::writer::file_writer::ParquetWriterBuilder; -//! # use iceberg::writer::file_writer::location_generator::{ -//! # DefaultFileNameGenerator, DefaultLocationGenerator, -//! # }; +//! # use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator; //! # use parquet::file::properties::WriterProperties; //! # use std::collections::HashMap; //! # #[tokio::main] @@ -336,7 +328,7 @@ //! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) //! # .await?; //! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; -//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let location_generator = table.location_generator()?; //! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); //! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); //! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(