From 9f4a353488727711de3d82ac8a6bdff87013782a Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Fri, 20 Nov 2026 14:40:13 +0000 Subject: [PATCH] refactor(writer)!: make location generators trait objects LocationGenerator no longer requires Clone, so a generator chosen at runtime can be an Arc. Writers store one instead of taking a location generator type parameter, and Table::location_generator returns the generator for the layout the table properties select. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + crates/catalog/s3tables/src/catalog.rs | 2 +- crates/iceberg/public-api.txt | 185 +++++++++--------- .../src/arrow/caching_delete_file_loader.rs | 4 +- crates/iceberg/src/cow_rewrite/writer.rs | 27 +-- crates/iceberg/src/table.rs | 16 ++ .../writer/base_writer/data_file_writer.rs | 30 ++- .../base_writer/equality_delete_writer.rs | 40 ++-- .../base_writer/position_delete_input.rs | 8 +- .../base_writer/position_delete_writer.rs | 39 ++-- .../writer/file_writer/location_generator.rs | 115 ++++++++++- .../src/writer/file_writer/rolling_writer.rs | 108 +++++++--- crates/iceberg/src/writer/mod.rs | 24 +-- .../writer/partitioning/clustered_writer.rs | 8 +- .../src/writer/partitioning/fanout_writer.rs | 6 +- .../partitioning/unpartitioned_writer.rs | 2 +- .../tests/conflict_commit_test.rs | 2 +- 17 files changed, 373 insertions(+), 245 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab34c96d1..27fdfa6b3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Breaking Changes +* `LocationGenerator` no longer requires `Clone`, so it can be used as `Arc`. Generic code that clones a `LocationGenerator` must add a `Clone` bound. `RollingFileWriterBuilder`, `RollingFileWriter`, `DataFileWriterBuilder`, `DataFileWriter`, `EqualityDeleteFileWriterBuilder`, `EqualityDeleteFileWriter`, `PositionDeleteFileWriterBuilder` and `PositionDeleteFileWriter` lose their location generator type parameter: drop it from type annotations, and wrap the generator passed to `RollingFileWriterBuilder::new` or `new_with_default_file_size` in `Arc::new(..)`. Use `Table::location_generator()` to get the generator for the table's layout. See [#3730](https://github.com/apache/iceberg-rust/pull/3730). + * `TableScan::to_arrow` now requires `ArrowReaderOptions`; pass `Default::default()` for default reader settings. Move reader-setting calls from `TableScanBuilder` and `ArrowReaderBuilder` to `ArrowReaderOptions::builder()`, then pass the options to `to_arrow(options)` or `ArrowReaderBuilder::with_options(options)`. Configure Parquet I/O through the nested `ParquetReadOptions`. Scan `with_concurrency_limit()` now bounds manifest planning only; set `with_data_file_concurrency_limit` on the reader-options builder to bound data-file and delete-file reads. Unset reader concurrency uses execution-host parallelism. See [#3514](https://github.com/apache/iceberg-rust/pull/3514). * `FileWrite::close`, `EncryptedOutputFile::write`, `ManifestListWriter::close`, and `PuffinWriter::close` now return `Result` instead of `Result<()>`. The returned size is the on-disk size, including encryption overhead. Custom `FileWrite` implementations must return the number of bytes stored. diff --git a/crates/catalog/s3tables/src/catalog.rs b/crates/catalog/s3tables/src/catalog.rs index c96d06abd4..3652c69a70 100644 --- a/crates/catalog/s3tables/src/catalog.rs +++ b/crates/catalog/s3tables/src/catalog.rs @@ -1322,7 +1322,7 @@ mod tests { let rw = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, table.file_io().clone(), - location_generator, + Arc::new(location_generator), file_name_generator, ); let mut writer = DataFileWriterBuilder::new(rw).build(None).await.unwrap(); diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 6dd6bd5f37..95d9d2c89e 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3483,6 +3483,7 @@ pub fn iceberg::table::Table::encryption_manager(&self) -> core::option::Option< pub fn iceberg::table::Table::file_io(&self) -> &iceberg::io::FileIO pub fn iceberg::table::Table::identifier(&self) -> &iceberg::TableIdent pub fn iceberg::table::Table::inspect(&self) -> iceberg::inspect::MetadataTable<'_> +pub fn iceberg::table::Table::location_generator(&self) -> iceberg::Result> pub fn iceberg::table::Table::manifest_list_reader(&self, snapshot: &iceberg::spec::SnapshotRef) -> iceberg::spec::ManifestListReader pub fn iceberg::table::Table::manifest_reader(&self) -> iceberg::spec::ManifestReader pub fn iceberg::table::Table::metadata(&self) -> &iceberg::spec::TableMetadata @@ -3606,39 +3607,39 @@ pub fn iceberg::util::snapshot::snapshot_id_as_of_time(table_metadata: &iceberg: pub mod iceberg::writer pub mod iceberg::writer::base_writer pub mod iceberg::writer::base_writer::data_file_writer -pub struct iceberg::writer::base_writer::data_file_writer::DataFileWriter -impl iceberg::writer::CurrentFileStatus for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_file_path(&self) -> alloc::string::String -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_row_num(&self) -> usize -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_written_size(&self) -> usize -impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl core::fmt::Debug for iceberg::writer::base_writer::data_file_writer::DataFileWriter -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder -impl iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder) -> Self -impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub type iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::R = iceberg::writer::base_writer::data_file_writer::DataFileWriter -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl core::fmt::Debug for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::data_file_writer::DataFileWriter +impl iceberg::writer::CurrentFileStatus for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_written_size(&self) -> usize +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::data_file_writer::DataFileWriter +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder +impl iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder) -> Self +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::R = iceberg::writer::base_writer::data_file_writer::DataFileWriter +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub mod iceberg::writer::base_writer::equality_delete_writer -pub struct iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter -impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder -impl iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder, config: iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig) -> Self -impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub type iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::R = iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder +impl iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder, config: iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig) -> Self +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::R = iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig impl iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig::new(equality_ids: alloc::vec::Vec, original_schema: iceberg::spec::SchemaRef) -> iceberg::Result @@ -3646,20 +3647,20 @@ pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWrite impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub mod iceberg::writer::base_writer::position_delete_writer -pub struct iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter -impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl core::fmt::Debug for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder -impl iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder) -> Self -impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub type iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::R = iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl core::fmt::Debug for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder +impl iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder) -> Self +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::R = iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn iceberg::writer::base_writer::position_delete_writer::position_delete_schema() -> iceberg::spec::SchemaRef pub mod iceberg::writer::file_writer pub mod iceberg::writer::file_writer::location_generator @@ -3695,32 +3696,32 @@ pub trait iceberg::writer::file_writer::location_generator::FileNameGenerator: c pub fn iceberg::writer::file_writer::location_generator::FileNameGenerator::generate_file_name(&self) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::FileNameGenerator for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator pub fn iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator::generate_file_name(&self) -> alloc::string::String -pub trait iceberg::writer::file_writer::location_generator::LocationGenerator: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static +pub trait iceberg::writer::file_writer::location_generator::LocationGenerator: core::marker::Send + core::marker::Sync + 'static pub fn iceberg::writer::file_writer::location_generator::LocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::DefaultLocationGenerator pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String pub mod iceberg::writer::file_writer::rolling_writer -pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriter -impl iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub async fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::close(self) -> iceberg::Result> -pub async fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::write(&mut self, partition_key: &core::option::Option, input: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> -impl core::fmt::Debug for iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::rolling_writer::RollingFileWriter -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_file_path(&self) -> alloc::string::String -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_row_num(&self) -> usize -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_written_size(&self) -> usize -pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder -impl iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::build(&self) -> iceberg::writer::file_writer::rolling_writer::RollingFileWriter -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::new(inner_builder: B, target_file_size: usize, file_io: iceberg::io::FileIO, location_generator: L, file_name_generator: F) -> Self -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::new_with_default_file_size(inner_builder: B, file_io: iceberg::io::FileIO, location_generator: L, file_name_generator: F) -> Self -impl core::clone::Clone for iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::clone(&self) -> iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder -impl core::fmt::Debug for iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriter +impl iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub async fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::close(self) -> iceberg::Result> +pub async fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::write(&mut self, partition_key: &core::option::Option, input: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> +impl core::fmt::Debug for iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::rolling_writer::RollingFileWriter +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_written_size(&self) -> usize +pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder +impl iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::build(&self) -> iceberg::writer::file_writer::rolling_writer::RollingFileWriter +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::new(inner_builder: B, target_file_size: usize, file_io: iceberg::io::FileIO, location_generator: alloc::sync::Arc, file_name_generator: F) -> Self +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::new_with_default_file_size(inner_builder: B, file_io: iceberg::io::FileIO, location_generator: alloc::sync::Arc, file_name_generator: F) -> Self +impl core::fmt::Debug for iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder + core::fmt::Debug, F: iceberg::writer::file_writer::location_generator::FileNameGenerator + core::fmt::Debug +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::clone::Clone for iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::clone(&self) -> iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder pub struct iceberg::writer::file_writer::ParquetWriter impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::ParquetWriter pub fn iceberg::writer::file_writer::ParquetWriter::current_file_path(&self) -> alloc::string::String @@ -3793,38 +3794,38 @@ impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::Parque pub fn iceberg::writer::file_writer::ParquetWriter::current_file_path(&self) -> alloc::string::String pub fn iceberg::writer::file_writer::ParquetWriter::current_row_num(&self) -> usize pub fn iceberg::writer::file_writer::ParquetWriter::current_written_size(&self) -> usize -impl iceberg::writer::CurrentFileStatus for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_file_path(&self) -> alloc::string::String -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_row_num(&self) -> usize -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_written_size(&self) -> usize -impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::rolling_writer::RollingFileWriter -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_file_path(&self) -> alloc::string::String -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_row_num(&self) -> usize -pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_written_size(&self) -> usize +impl iceberg::writer::CurrentFileStatus for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_written_size(&self) -> usize +impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::rolling_writer::RollingFileWriter +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_written_size(&self) -> usize pub trait iceberg::writer::IcebergWriter: core::marker::Send + 'static pub fn iceberg::writer::IcebergWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub fn iceberg::writer::IcebergWriter::write<'life0, 'async_trait>(&'life0 mut self, input: I) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub trait iceberg::writer::IcebergWriterBuilder: core::marker::Send + core::marker::Sync + 'static pub type iceberg::writer::IcebergWriterBuilder::R: iceberg::writer::IcebergWriter pub fn iceberg::writer::IcebergWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub type iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::R = iceberg::writer::base_writer::data_file_writer::DataFileWriter -pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub type iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::R = iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter -pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait -impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator -pub type iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::R = iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter -pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::R = iceberg::writer::base_writer::data_file_writer::DataFileWriter +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::R = iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::R = iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub macro iceberg::ensure_data_valid! #[non_exhaustive] pub enum iceberg::ErrorKind pub iceberg::ErrorKind::CatalogCommitConflicts diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index dc5690248e..2d2c1b4341 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -2738,7 +2738,9 @@ mod tests { let rolling = RollingFileWriterBuilder::new_with_default_file_size( ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(delete_schema)), FileIO::new_with_fs(), - DefaultLocationGenerator::with_data_location(dir.path().to_str().unwrap().to_string()), + Arc::new(DefaultLocationGenerator::with_data_location( + dir.path().to_str().unwrap().to_string(), + )), DefaultFileNameGenerator::new("eq-del".to_string(), None, DataFileFormat::Parquet), ); let mut writer = EqualityDeleteFileWriterBuilder::new(rolling, config) diff --git a/crates/iceberg/src/cow_rewrite/writer.rs b/crates/iceberg/src/cow_rewrite/writer.rs index ce235e889b..6d8fdb8a3c 100644 --- a/crates/iceberg/src/cow_rewrite/writer.rs +++ b/crates/iceberg/src/cow_rewrite/writer.rs @@ -27,10 +27,7 @@ use crate::table::Table; use crate::writer::IcebergWriterBuilder; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; use crate::writer::file_writer::ParquetWriterBuilder; -use crate::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator, - ObjectStorageLocationGenerator, -}; +use crate::writer::file_writer::location_generator::DefaultFileNameGenerator; use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; /// Builds a boxed replacement-data-file writer. @@ -44,9 +41,8 @@ use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; /// batch is written, so it is safe to construct one optimistically and only /// write to it once a source file is known to have changed. /// -/// The location layout follows `write.object-storage.enabled`: tables that opt -/// into the object-storage layout get hash-entropy paths from -/// [`ObjectStorageLocationGenerator`], everything else gets the default layout. +/// File locations follow the layout selected by the table properties; see +/// [`Table::location_generator`]. pub(crate) async fn build_replacement_writer( table: &Table, write_schema: SchemaRef, @@ -72,27 +68,12 @@ pub(crate) async fn build_replacement_writer( parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone()); } - if table_props.write_object_storage_enabled()? { - let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?; - build_data_file_writer(table, parquet_builder, location_generator, partition_key).await - } else { - let location_generator = DefaultLocationGenerator::new(table.metadata())?; - build_data_file_writer(table, parquet_builder, location_generator, partition_key).await - } -} - -async fn build_data_file_writer( - table: &Table, - parquet_builder: ParquetWriterBuilder, - location_generator: L, - partition_key: Option, -) -> Result> { + let location_generator = table.location_generator()?; let file_name_generator = DefaultFileNameGenerator::new( format!("cow-rewrite-{}", Uuid::now_v7()), None, DataFileFormat::Parquet, ); - let table_props = table.metadata().table_properties(); let rolling_builder = RollingFileWriterBuilder::new( parquet_builder, table_props.write_target_file_size_bytes()?, diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index d2dded58a5..56e7d21807 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -31,6 +31,7 @@ use crate::scan::TableScanBuilder; use crate::spec::{ ManifestListReader, ManifestReader, SchemaRef, SnapshotRef, TableMetadata, TableMetadataRef, }; +use crate::writer::file_writer::location_generator::{LocationGenerator, table_location_generator}; use crate::{Result, TableIdent}; /// Builder to create table scan. @@ -334,6 +335,21 @@ impl Table { pub fn reader_builder(&self) -> ArrowReaderBuilder { ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone()) } + + /// Creates the location generator for the layout selected by the table properties. + /// + /// The layout follows + /// [`write.object-storage.enabled`](crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED); + /// see [`ObjectStorageLocationGenerator`](crate::writer::file_writer::location_generator::ObjectStorageLocationGenerator) + /// and [`DefaultLocationGenerator`](crate::writer::file_writer::location_generator::DefaultLocationGenerator). + /// Java's `write.location-provider.impl` is not supported and is ignored. + /// + /// # Errors + /// + /// Returns an error if a table property that determines the layout has an invalid value. + pub fn location_generator(&self) -> Result> { + table_location_generator(self.metadata()) + } } /// `StaticTable` is a read-only table struct that can be created from a metadata file or from `TableMetaData` without a catalog. diff --git a/crates/iceberg/src/writer/base_writer/data_file_writer.rs b/crates/iceberg/src/writer/base_writer/data_file_writer.rs index c7a988374e..d61b5e7f04 100644 --- a/crates/iceberg/src/writer/base_writer/data_file_writer.rs +++ b/crates/iceberg/src/writer/base_writer/data_file_writer.rs @@ -22,37 +22,35 @@ use arrow_array::RecordBatch; use crate::error::invalid_data; use crate::spec::{DataContentType, DataFile, PartitionKey}; use crate::writer::file_writer::FileWriterBuilder; -use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; +use crate::writer::file_writer::location_generator::FileNameGenerator; use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder}; use crate::writer::{CurrentFileStatus, IcebergWriter, IcebergWriterBuilder}; use crate::{Error, ErrorKind, Result}; /// Builder for `DataFileWriter`. #[derive(Debug)] -pub struct DataFileWriterBuilder { - inner: RollingFileWriterBuilder, +pub struct DataFileWriterBuilder { + inner: RollingFileWriterBuilder, } -impl DataFileWriterBuilder +impl DataFileWriterBuilder where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { /// Create a new `DataFileWriterBuilder` using a `RollingFileWriterBuilder`. - pub fn new(inner: RollingFileWriterBuilder) -> Self { + pub fn new(inner: RollingFileWriterBuilder) -> Self { Self { inner } } } #[async_trait::async_trait] -impl IcebergWriterBuilder for DataFileWriterBuilder +impl IcebergWriterBuilder for DataFileWriterBuilder where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { - type R = DataFileWriter; + type R = DataFileWriter; async fn build(&self, partition_key: Option) -> Result { Ok(DataFileWriter { @@ -64,16 +62,15 @@ where /// A writer write data is within one spec/partition. #[derive(Debug)] -pub struct DataFileWriter { - inner: Option>, +pub struct DataFileWriter { + inner: Option>, partition_key: Option, } #[async_trait::async_trait] -impl IcebergWriter for DataFileWriter +impl IcebergWriter for DataFileWriter where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { async fn write(&mut self, batch: RecordBatch) -> Result<()> { @@ -112,10 +109,9 @@ where } } -impl CurrentFileStatus for DataFileWriter +impl CurrentFileStatus for DataFileWriter where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { fn current_file_path(&self) -> String { @@ -180,7 +176,7 @@ mod test { let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( pw, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); @@ -266,7 +262,7 @@ mod test { let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); diff --git a/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs b/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs index f62016addc..fbbbb126bc 100644 --- a/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs +++ b/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs @@ -30,33 +30,25 @@ use crate::arrow::schema_to_arrow_schema; use crate::error::invalid_data; use crate::spec::{DataFile, PartitionKey, Schema, SchemaRef}; use crate::writer::file_writer::FileWriterBuilder; -use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; +use crate::writer::file_writer::location_generator::FileNameGenerator; use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder}; use crate::writer::{IcebergWriter, IcebergWriterBuilder}; use crate::{Error, ErrorKind, Result}; /// Builder for `EqualityDeleteWriter`. #[derive(Debug)] -pub struct EqualityDeleteFileWriterBuilder< - B: FileWriterBuilder, - L: LocationGenerator, - F: FileNameGenerator, -> { - inner: RollingFileWriterBuilder, +pub struct EqualityDeleteFileWriterBuilder { + inner: RollingFileWriterBuilder, config: EqualityDeleteWriterConfig, } -impl EqualityDeleteFileWriterBuilder +impl EqualityDeleteFileWriterBuilder where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { /// Create a new `EqualityDeleteFileWriterBuilder` using a `RollingFileWriterBuilder`. - pub fn new( - inner: RollingFileWriterBuilder, - config: EqualityDeleteWriterConfig, - ) -> Self { + pub fn new(inner: RollingFileWriterBuilder, config: EqualityDeleteWriterConfig) -> Self { Self { inner, config } } } @@ -149,13 +141,12 @@ impl EqualityDeleteWriterConfig { } #[async_trait::async_trait] -impl IcebergWriterBuilder for EqualityDeleteFileWriterBuilder +impl IcebergWriterBuilder for EqualityDeleteFileWriterBuilder where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { - type R = EqualityDeleteFileWriter; + type R = EqualityDeleteFileWriter; async fn build(&self, partition_key: Option) -> Result { Ok(EqualityDeleteFileWriter { @@ -169,22 +160,17 @@ where /// Writer used to write equality delete files. #[derive(Debug)] -pub struct EqualityDeleteFileWriter< - B: FileWriterBuilder, - L: LocationGenerator, - F: FileNameGenerator, -> { - inner: Option>, +pub struct EqualityDeleteFileWriter { + inner: Option>, projector: RecordBatchProjector, equality_ids: Vec, partition_key: Option, } #[async_trait::async_trait] -impl IcebergWriter for EqualityDeleteFileWriter +impl IcebergWriter for EqualityDeleteFileWriter where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { async fn write(&mut self, batch: RecordBatch) -> Result<()> { @@ -464,7 +450,7 @@ mod test { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( pb, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); let mut equality_delete_writer = @@ -685,7 +671,7 @@ mod test { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( pb, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); let mut equality_delete_writer = @@ -814,7 +800,7 @@ mod test { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( pb, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); let mut equality_delete_writer = diff --git a/crates/iceberg/src/writer/base_writer/position_delete_input.rs b/crates/iceberg/src/writer/base_writer/position_delete_input.rs index 483739f685..6898adf8af 100644 --- a/crates/iceberg/src/writer/base_writer/position_delete_input.rs +++ b/crates/iceberg/src/writer/base_writer/position_delete_input.rs @@ -591,11 +591,7 @@ mod test { /// configured with the [`position_delete_schema`]. fn writer_setup( temp_dir: &TempDir, - ) -> PositionDeleteFileWriterBuilder< - ParquetWriterBuilder, - DefaultLocationGenerator, - DefaultFileNameGenerator, - > { + ) -> PositionDeleteFileWriterBuilder { let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), @@ -609,7 +605,7 @@ mod test { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io, - location_gen, + Arc::new(location_gen), file_name_gen, ); PositionDeleteFileWriterBuilder::new(rolling_writer_builder) diff --git a/crates/iceberg/src/writer/base_writer/position_delete_writer.rs b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs index 3b023c9c62..097d40f4fd 100644 --- a/crates/iceberg/src/writer/base_writer/position_delete_writer.rs +++ b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs @@ -45,7 +45,7 @@ use crate::metadata_columns::{ }; use crate::spec::{DataContentType, DataFile, PartitionKey, Schema, SchemaRef}; use crate::writer::file_writer::FileWriterBuilder; -use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; +use crate::writer::file_writer::location_generator::FileNameGenerator; use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder}; use crate::writer::{IcebergWriter, IcebergWriterBuilder}; use crate::{Error, ErrorKind, Result}; @@ -171,18 +171,13 @@ fn validate_position_delete_batch(batch: &RecordBatch) -> Result<()> { /// Builder for [`PositionDeleteFileWriter`]. #[derive(Debug)] -pub struct PositionDeleteFileWriterBuilder< - B: FileWriterBuilder, - L: LocationGenerator, - F: FileNameGenerator, -> { - inner: RollingFileWriterBuilder, +pub struct PositionDeleteFileWriterBuilder { + inner: RollingFileWriterBuilder, } -impl PositionDeleteFileWriterBuilder +impl PositionDeleteFileWriterBuilder where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { /// Create a new `PositionDeleteFileWriterBuilder` using a `RollingFileWriterBuilder`. @@ -191,19 +186,18 @@ where /// with the [`position_delete_schema`]; the per-batch validation in /// [`PositionDeleteFileWriter::write`] guards against a mismatched batch, but /// the caller is responsible for wiring the same schema into the file writer. - pub fn new(inner: RollingFileWriterBuilder) -> Self { + pub fn new(inner: RollingFileWriterBuilder) -> Self { Self { inner } } } #[async_trait::async_trait] -impl IcebergWriterBuilder for PositionDeleteFileWriterBuilder +impl IcebergWriterBuilder for PositionDeleteFileWriterBuilder where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { - type R = PositionDeleteFileWriter; + type R = PositionDeleteFileWriter; async fn build(&self, partition_key: Option) -> Result { Ok(PositionDeleteFileWriter { @@ -215,20 +209,15 @@ where /// Writer used to write position delete files within one spec/partition. #[derive(Debug)] -pub struct PositionDeleteFileWriter< - B: FileWriterBuilder, - L: LocationGenerator, - F: FileNameGenerator, -> { - inner: Option>, +pub struct PositionDeleteFileWriter { + inner: Option>, partition_key: Option, } #[async_trait::async_trait] -impl IcebergWriter for PositionDeleteFileWriter +impl IcebergWriter for PositionDeleteFileWriter where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { /// Writes a batch of `(file_path, pos)` records; the shape is validated on every @@ -353,11 +342,7 @@ mod test { temp_dir: &TempDir, ) -> ( FileIO, - PositionDeleteFileWriterBuilder< - ParquetWriterBuilder, - DefaultLocationGenerator, - DefaultFileNameGenerator, - >, + PositionDeleteFileWriterBuilder, ) { let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( @@ -373,7 +358,7 @@ mod test { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); ( diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 26207d3e1d..bbbf4c78a8 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -25,7 +25,7 @@ use crate::spec::{DataFileFormat, PartitionKey, TableMetadata}; use crate::util::location::strip_trailing_slash; /// `LocationGenerator` used to generate the location of data file. -pub trait LocationGenerator: Clone + Send + Sync + 'static { +pub trait LocationGenerator: Send + Sync + 'static { /// Generate an absolute path for the given file name that includes the partition path. /// /// # Arguments @@ -54,6 +54,11 @@ const ENTROPY_DIR_DEPTH: usize = 3; #[derive(Clone, Debug)] /// `DefaultLocationGenerator` used to generate the data dir location of data file. /// The location is generated based on the table location and the data location in table properties. +/// +/// Paths use `{data location}/{partition path}/{file name}`, omitting the +/// partition path for unpartitioned files. It ignores `write.object-storage.enabled`; use +/// [`Table::location_generator`](crate::table::Table::location_generator) for the layout the +/// table properties select. pub struct DefaultLocationGenerator { data_location: String, } @@ -203,6 +208,22 @@ impl LocationGenerator for ObjectStorageLocationGenerator { } } +/// Creates the generator for the layout selected by the table properties. +pub(crate) fn table_location_generator( + table_metadata: &TableMetadata, +) -> Result> { + if table_metadata + .table_properties() + .write_object_storage_enabled()? + { + Ok(Arc::new(ObjectStorageLocationGenerator::new( + table_metadata, + )?)) + } else { + Ok(Arc::new(DefaultLocationGenerator::new(table_metadata)?)) + } +} + /// Derive the `{parent}/{name}` context from a table location, mirroring Hadoop's /// `Path.getParent().getName()` / `Path.getName()`. fn path_context(table_location: &str) -> String { @@ -291,15 +312,18 @@ pub(crate) mod test { use std::collections::HashMap; use std::sync::Arc; + use rstest::rstest; use uuid::Uuid; use super::LocationGenerator; + use crate::ErrorKind; use crate::spec::{ FormatVersion, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Schema, Struct, StructType, TableMetadata, TableProperties, Transform, Type, }; use crate::writer::file_writer::location_generator::{ DefaultLocationGenerator, FileNameGenerator, ObjectStorageLocationGenerator, + table_location_generator, }; #[test] @@ -609,6 +633,95 @@ pub(crate) mod test { ); } + #[rstest] + #[case::unset(&[], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::disabled(&[(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, "false")], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::java_location_provider(&[("write.location-provider.impl", "com.example.Provider")], [ + "s3://data.db/table/data/test.parquet", + "s3://data.db/table/data/data%231=val%231/test.parquet", + ])] + #[case::enabled(&[(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, "true")], [ + "s3://data.db/table/data/0110/1010/0011/11101000/test.parquet", + "s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet", + ])] + fn test_table_location_follows_properties( + #[case] properties: &[(&str, &str)], + #[case] expected: [&str; 2], + ) { + let metadata = table_metadata_with( + "s3://data.db/table", + properties + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "data#1", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("data#1", "data#1", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition_key = PartitionKey::new( + partition_spec, + schema, + Struct::from_iter([Some(Literal::string("val#1"))]), + ); + let generator = table_location_generator(&metadata).unwrap(); + + assert_eq!( + [ + generator.generate_location(None, "test.parquet"), + generator.generate_location(Some(&partition_key), "test.parquet"), + ], + expected + ); + } + + #[test] + fn test_table_location_rejects_invalid_property() { + let metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED.to_string(), + "yes".to_string(), + )]), + ); + let err = table_location_generator(&metadata).err().unwrap(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_default_location_ignores_object_storage_property() { + let metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED.to_string(), + "true".to_string(), + )]), + ); + let generator = DefaultLocationGenerator::new(&metadata).unwrap(); + + assert_eq!( + generator.generate_location(None, "test.parquet"), + "s3://data.db/table/data/test.parquet" + ); + } + /// Build a minimal `TableMetadata` for location generator tests. fn table_metadata_with(location: &str, properties: HashMap) -> TableMetadata { TableMetadata { diff --git a/crates/iceberg/src/writer/file_writer/rolling_writer.rs b/crates/iceberg/src/writer/file_writer/rolling_writer.rs index 9a45cd0fb2..bec68e822c 100644 --- a/crates/iceberg/src/writer/file_writer/rolling_writer.rs +++ b/crates/iceberg/src/writer/file_writer/rolling_writer.rs @@ -16,6 +16,7 @@ // under the License. use std::fmt::{Debug, Formatter}; +use std::sync::Arc; use arrow_array::RecordBatch; @@ -27,23 +28,33 @@ use crate::writer::file_writer::{FileWriter, FileWriterBuilder}; use crate::{Error, ErrorKind, Result}; /// Builder for [`RollingFileWriter`]. -#[derive(Clone, Debug)] -pub struct RollingFileWriterBuilder< - B: FileWriterBuilder, - L: LocationGenerator, - F: FileNameGenerator, -> { +#[derive(Clone)] +pub struct RollingFileWriterBuilder { inner_builder: B, target_file_size: usize, file_io: FileIO, - location_generator: L, + location_generator: Arc, file_name_generator: F, } -impl RollingFileWriterBuilder +impl Debug for RollingFileWriterBuilder +where + B: FileWriterBuilder + Debug, + F: FileNameGenerator + Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RollingFileWriterBuilder") + .field("inner_builder", &self.inner_builder) + .field("target_file_size", &self.target_file_size) + .field("file_io", &self.file_io) + .field("file_name_generator", &self.file_name_generator) + .finish_non_exhaustive() + } +} + +impl RollingFileWriterBuilder where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { /// Creates a new `RollingFileWriterBuilder` with the specified target file size. @@ -63,7 +74,7 @@ where inner_builder: B, target_file_size: usize, file_io: FileIO, - location_generator: L, + location_generator: Arc, file_name_generator: F, ) -> Self { Self { @@ -90,7 +101,7 @@ where pub fn new_with_default_file_size( inner_builder: B, file_io: FileIO, - location_generator: L, + location_generator: Arc, file_name_generator: F, ) -> Self { Self { @@ -103,7 +114,7 @@ where } /// Build a new [`RollingFileWriter`]. - pub fn build(&self) -> RollingFileWriter { + pub fn build(&self) -> RollingFileWriter { RollingFileWriter { inner: None, inner_builder: self.inner_builder.clone(), @@ -122,20 +133,19 @@ where /// This writer wraps another file writer that tracks the amount of data written. /// When the data size exceeds the target size, it closes the current file and /// starts writing to a new one. -pub struct RollingFileWriter { +pub struct RollingFileWriter { inner: Option, inner_builder: B, target_file_size: usize, data_file_builders: Vec, file_io: FileIO, - location_generator: L, + location_generator: Arc, file_name_generator: F, } -impl Debug for RollingFileWriter +impl Debug for RollingFileWriter where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { @@ -146,10 +156,9 @@ where } } -impl RollingFileWriter +impl RollingFileWriter where B: FileWriterBuilder, - L: LocationGenerator, F: FileNameGenerator, { /// Determines if the writer should roll over to a new file. @@ -239,9 +248,7 @@ where } } -impl CurrentFileStatus - for RollingFileWriter -{ +impl CurrentFileStatus for RollingFileWriter { fn current_file_path(&self) -> String { self.inner.as_ref().unwrap().current_file_path() } @@ -271,7 +278,7 @@ mod tests { use crate::arrow::test_utils::read_encrypted_parquet; use crate::encryption::StandardKeyMetadata; use crate::io::FileIO; - use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Schema, Type}; + use crate::spec::{DataFileFormat, NestedField, PartitionKey, PrimitiveType, Schema, Type}; use crate::test_utils::make_encryption_manager; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; use crate::writer::file_writer::ParquetWriterBuilder; @@ -304,6 +311,57 @@ mod tests { ]) } + /// Puts every file directly under `memory:///custom`. + struct FixedDirectory; + + impl LocationGenerator for FixedDirectory { + fn generate_location(&self, _: Option<&PartitionKey>, file_name: &str) -> String { + format!("memory:///custom/{file_name}") + } + } + + /// Writes one row with `builder` and returns the data file's path. + async fn written_path( + builder: RollingFileWriterBuilder, + ) -> Result { + let batch = RecordBatch::try_new(Arc::new(make_test_arrow_schema()), vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(StringArray::from(vec!["Alice"])), + ])?; + let mut writer = DataFileWriterBuilder::new(builder).build(None).await?; + writer.write(batch).await?; + let data_files = writer.close().await?; + Ok(data_files[0].file_path().to_string()) + } + + #[tokio::test] + async fn test_rolling_writer_uses_given_location_generator() -> Result<()> { + let schema = Arc::new(make_test_schema()?); + let new = RollingFileWriterBuilder::new( + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()), + 1024 * 1024, + FileIO::new_with_memory(), + Arc::new(FixedDirectory), + DefaultFileNameGenerator::new("new".to_string(), None, DataFileFormat::Parquet), + ); + let default_size = RollingFileWriterBuilder::new_with_default_file_size( + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema), + FileIO::new_with_memory(), + Arc::new(FixedDirectory), + DefaultFileNameGenerator::new("default".to_string(), None, DataFileFormat::Parquet), + ); + + assert_eq!( + written_path(new).await?, + "memory:///custom/new-00000.parquet" + ); + assert_eq!( + written_path(default_size).await?, + "memory:///custom/default-00000.parquet" + ); + Ok(()) + } + #[tokio::test] async fn test_rolling_writer_basic() -> Result<()> { let temp_dir = TempDir::new()?; @@ -326,7 +384,7 @@ mod tests { parquet_writer_builder, 1024 * 1024, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); @@ -384,7 +442,7 @@ mod tests { parquet_writer_builder, 1024, file_io, - location_gen, + Arc::new(location_gen), file_name_gen, ); @@ -470,7 +528,7 @@ mod tests { parquet_writer_builder, 1024, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); diff --git a/crates/iceberg/src/writer/mod.rs b/crates/iceberg/src/writer/mod.rs index ca401097ab..0a3850ee81 100644 --- a/crates/iceberg/src/writer/mod.rs +++ b/crates/iceberg/src/writer/mod.rs @@ -50,9 +50,7 @@ //! use iceberg::transaction::Transaction; //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; -//! use iceberg::writer::file_writer::location_generator::{ -//! DefaultFileNameGenerator, DefaultLocationGenerator, -//! }; +//! use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent}; //! use parquet::file::properties::WriterProperties; @@ -78,7 +76,7 @@ //! let table = catalog //! .load_table(&TableIdent::from_strs(["hello", "world"])?) //! .await?; -//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! let location_generator = table.location_generator()?; //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -124,9 +122,7 @@ //! use iceberg::spec::{DataFile, PartitionKey}; //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; -//! use iceberg::writer::file_writer::location_generator::{ -//! DefaultFileNameGenerator, DefaultLocationGenerator, -//! }; +//! use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator; //! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; //! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent}; //! use parquet::file::properties::WriterProperties; @@ -207,7 +203,7 @@ //! table.metadata().current_schema().clone(), //! Struct::from_iter(vec![Some(Literal::string("Seattle"))]), //! ); -//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! let location_generator = table.location_generator()?; //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -261,9 +257,7 @@ //! # use iceberg::{Catalog, CatalogBuilder, Result, TableIdent}; //! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! # use iceberg::writer::file_writer::ParquetWriterBuilder; -//! # use iceberg::writer::file_writer::location_generator::{ -//! # DefaultFileNameGenerator, DefaultLocationGenerator, -//! # }; +//! # use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator; //! # use parquet::file::properties::WriterProperties; //! # use std::collections::HashMap; //! # #[tokio::main] @@ -272,7 +266,7 @@ //! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) //! # .await?; //! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; -//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let location_generator = table.location_generator()?; //! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); //! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); //! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( @@ -325,9 +319,7 @@ //! # use iceberg::{Catalog, CatalogBuilder, Result, TableIdent}; //! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! # use iceberg::writer::file_writer::ParquetWriterBuilder; -//! # use iceberg::writer::file_writer::location_generator::{ -//! # DefaultFileNameGenerator, DefaultLocationGenerator, -//! # }; +//! # use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator; //! # use parquet::file::properties::WriterProperties; //! # use std::collections::HashMap; //! # #[tokio::main] @@ -336,7 +328,7 @@ //! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) //! # .await?; //! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; -//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let location_generator = table.location_generator()?; //! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); //! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); //! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( diff --git a/crates/iceberg/src/writer/partitioning/clustered_writer.rs b/crates/iceberg/src/writer/partitioning/clustered_writer.rs index 21b52bd813..35ba355e4f 100644 --- a/crates/iceberg/src/writer/partitioning/clustered_writer.rs +++ b/crates/iceberg/src/writer/partitioning/clustered_writer.rs @@ -199,7 +199,7 @@ mod tests { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); @@ -314,7 +314,7 @@ mod tests { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); @@ -445,7 +445,7 @@ mod tests { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); @@ -550,7 +550,7 @@ mod tests { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); diff --git a/crates/iceberg/src/writer/partitioning/fanout_writer.rs b/crates/iceberg/src/writer/partitioning/fanout_writer.rs index 2cdd865375..ee95c8c9ba 100644 --- a/crates/iceberg/src/writer/partitioning/fanout_writer.rs +++ b/crates/iceberg/src/writer/partitioning/fanout_writer.rs @@ -176,7 +176,7 @@ mod tests { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); @@ -291,7 +291,7 @@ mod tests { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); @@ -425,7 +425,7 @@ mod tests { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io.clone(), - location_gen, + Arc::new(location_gen), file_name_gen, ); let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); diff --git a/crates/iceberg/src/writer/partitioning/unpartitioned_writer.rs b/crates/iceberg/src/writer/partitioning/unpartitioned_writer.rs index de127e0c9d..8dbd910180 100644 --- a/crates/iceberg/src/writer/partitioning/unpartitioned_writer.rs +++ b/crates/iceberg/src/writer/partitioning/unpartitioned_writer.rs @@ -163,7 +163,7 @@ mod tests { let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, file_io, - location_gen, + Arc::new(location_gen), file_name_gen, ); let writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); diff --git a/crates/integration_tests/tests/conflict_commit_test.rs b/crates/integration_tests/tests/conflict_commit_test.rs index 73da9736ef..af3324b8b7 100644 --- a/crates/integration_tests/tests/conflict_commit_test.rs +++ b/crates/integration_tests/tests/conflict_commit_test.rs @@ -83,7 +83,7 @@ async fn test_append_data_file_conflict() { let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( parquet_writer_builder, table.file_io().clone(), - location_generator.clone(), + Arc::new(location_generator), file_name_generator.clone(), ); let data_file_writer_builder = DataFileWriterBuilder::new(rolling_file_writer_builder);