Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/).

### Breaking Changes

* `LocationGenerator` no longer requires `Clone`, so it can be used as `Arc<dyn LocationGenerator>`. Generic code that clones a `LocationGenerator` must add a `Clone` bound. `RollingFileWriterBuilder`, `RollingFileWriter`, `DataFileWriterBuilder`, `DataFileWriter`, `EqualityDeleteFileWriterBuilder`, `EqualityDeleteFileWriter`, `PositionDeleteFileWriterBuilder` and `PositionDeleteFileWriter` lose their location generator type parameter: drop it from type annotations, and wrap the generator passed to `RollingFileWriterBuilder::new` or `new_with_default_file_size` in `Arc::new(..)`. Use `Table::location_generator()` to get the generator for the table's layout. See [#3730](https://github.com/apache/iceberg-rust/pull/3730).

* `TableScan::to_arrow` now requires `ArrowReaderOptions`; pass `Default::default()` for default reader settings. Move reader-setting calls from `TableScanBuilder` and `ArrowReaderBuilder` to `ArrowReaderOptions::builder()`, then pass the options to `to_arrow(options)` or `ArrowReaderBuilder::with_options(options)`. Configure Parquet I/O through the nested `ParquetReadOptions`. Scan `with_concurrency_limit()` now bounds manifest planning only; set `with_data_file_concurrency_limit` on the reader-options builder to bound data-file and delete-file reads. Unset reader concurrency uses execution-host parallelism. See [#3514](https://github.com/apache/iceberg-rust/pull/3514).

* `FileWrite::close`, `EncryptedOutputFile::write`, `ManifestListWriter::close`, and `PuffinWriter::close` now return `Result<io::FileMetadata>` instead of `Result<()>`. The returned size is the on-disk size, including encryption overhead. Custom `FileWrite` implementations must return the number of bytes stored.
Expand Down
2 changes: 1 addition & 1 deletion crates/catalog/s3tables/src/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1322,7 +1322,7 @@ mod tests {
let rw = RollingFileWriterBuilder::new_with_default_file_size(
parquet_writer_builder,
table.file_io().clone(),
location_generator,
Arc::new(location_generator),
file_name_generator,
);
let mut writer = DataFileWriterBuilder::new(rw).build(None).await.unwrap();
Expand Down
185 changes: 93 additions & 92 deletions crates/iceberg/public-api.txt

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion crates/iceberg/src/arrow/caching_delete_file_loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2738,7 +2738,9 @@ mod tests {
let rolling = RollingFileWriterBuilder::new_with_default_file_size(
ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(delete_schema)),
FileIO::new_with_fs(),
DefaultLocationGenerator::with_data_location(dir.path().to_str().unwrap().to_string()),
Arc::new(DefaultLocationGenerator::with_data_location(
dir.path().to_str().unwrap().to_string(),
)),
DefaultFileNameGenerator::new("eq-del".to_string(), None, DataFileFormat::Parquet),
);
let mut writer = EqualityDeleteFileWriterBuilder::new(rolling, config)
Expand Down
27 changes: 4 additions & 23 deletions crates/iceberg/src/cow_rewrite/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,7 @@ use crate::table::Table;
use crate::writer::IcebergWriterBuilder;
use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder;
use crate::writer::file_writer::ParquetWriterBuilder;
use crate::writer::file_writer::location_generator::{
DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator,
ObjectStorageLocationGenerator,
};
use crate::writer::file_writer::location_generator::DefaultFileNameGenerator;
use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder;

/// Builds a boxed replacement-data-file writer.
Expand All @@ -44,9 +41,8 @@ use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
/// batch is written, so it is safe to construct one optimistically and only
/// write to it once a source file is known to have changed.
///
/// The location layout follows `write.object-storage.enabled`: tables that opt
/// into the object-storage layout get hash-entropy paths from
/// [`ObjectStorageLocationGenerator`], everything else gets the default layout.
/// File locations follow the layout selected by the table properties; see
/// [`Table::location_generator`].
pub(crate) async fn build_replacement_writer(
table: &Table,
write_schema: SchemaRef,
Expand All @@ -72,27 +68,12 @@ pub(crate) async fn build_replacement_writer(
parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone());
}

if table_props.write_object_storage_enabled()? {
let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?;
build_data_file_writer(table, parquet_builder, location_generator, partition_key).await
} else {
let location_generator = DefaultLocationGenerator::new(table.metadata())?;
build_data_file_writer(table, parquet_builder, location_generator, partition_key).await
}
}

async fn build_data_file_writer<L: LocationGenerator>(
table: &Table,
parquet_builder: ParquetWriterBuilder,
location_generator: L,
partition_key: Option<PartitionKey>,
) -> Result<Box<dyn crate::writer::IcebergWriter>> {
let location_generator = table.location_generator()?;
let file_name_generator = DefaultFileNameGenerator::new(
format!("cow-rewrite-{}", Uuid::now_v7()),
None,
DataFileFormat::Parquet,
);
let table_props = table.metadata().table_properties();
let rolling_builder = RollingFileWriterBuilder::new(
parquet_builder,
table_props.write_target_file_size_bytes()?,
Expand Down
16 changes: 16 additions & 0 deletions crates/iceberg/src/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ use crate::scan::TableScanBuilder;
use crate::spec::{
ManifestListReader, ManifestReader, SchemaRef, SnapshotRef, TableMetadata, TableMetadataRef,
};
use crate::writer::file_writer::location_generator::{LocationGenerator, table_location_generator};
use crate::{Result, TableIdent};

/// Builder to create table scan.
Expand Down Expand Up @@ -334,6 +335,21 @@ impl Table {
pub fn reader_builder(&self) -> ArrowReaderBuilder {
ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone())
}

/// Creates the location generator for the layout selected by the table properties.
///
/// The layout follows
/// [`write.object-storage.enabled`](crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED);
/// see [`ObjectStorageLocationGenerator`](crate::writer::file_writer::location_generator::ObjectStorageLocationGenerator)
/// and [`DefaultLocationGenerator`](crate::writer::file_writer::location_generator::DefaultLocationGenerator).
/// Java's `write.location-provider.impl` is not supported and is ignored.
Comment on lines +341 to +345

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changes from #3730 are included here to show what that flow would look like with this design instead. This adds custom location providers on top.

///
/// # Errors
///
/// Returns an error if a table property that determines the layout has an invalid value.
pub fn location_generator(&self) -> Result<Arc<dyn LocationGenerator>> {
table_location_generator(self.metadata())
}
}

/// `StaticTable` is a read-only table struct that can be created from a metadata file or from `TableMetaData` without a catalog.
Expand Down
30 changes: 13 additions & 17 deletions crates/iceberg/src/writer/base_writer/data_file_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,37 +22,35 @@ use arrow_array::RecordBatch;
use crate::error::invalid_data;
use crate::spec::{DataContentType, DataFile, PartitionKey};
use crate::writer::file_writer::FileWriterBuilder;
use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator};
use crate::writer::file_writer::location_generator::FileNameGenerator;
use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder};
use crate::writer::{CurrentFileStatus, IcebergWriter, IcebergWriterBuilder};
use crate::{Error, ErrorKind, Result};

/// Builder for `DataFileWriter`.
#[derive(Debug)]
pub struct DataFileWriterBuilder<B: FileWriterBuilder, L: LocationGenerator, F: FileNameGenerator> {
inner: RollingFileWriterBuilder<B, L, F>,
pub struct DataFileWriterBuilder<B: FileWriterBuilder, F: FileNameGenerator> {
inner: RollingFileWriterBuilder<B, F>,
}

impl<B, L, F> DataFileWriterBuilder<B, L, F>
impl<B, F> DataFileWriterBuilder<B, F>
where
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
{
/// Create a new `DataFileWriterBuilder` using a `RollingFileWriterBuilder`.
pub fn new(inner: RollingFileWriterBuilder<B, L, F>) -> Self {
pub fn new(inner: RollingFileWriterBuilder<B, F>) -> Self {
Self { inner }
}
}

#[async_trait::async_trait]
impl<B, L, F> IcebergWriterBuilder for DataFileWriterBuilder<B, L, F>
impl<B, F> IcebergWriterBuilder for DataFileWriterBuilder<B, F>
where
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
{
type R = DataFileWriter<B, L, F>;
type R = DataFileWriter<B, F>;

async fn build(&self, partition_key: Option<PartitionKey>) -> Result<Self::R> {
Ok(DataFileWriter {
Expand All @@ -64,16 +62,15 @@ where

/// A writer write data is within one spec/partition.
#[derive(Debug)]
pub struct DataFileWriter<B: FileWriterBuilder, L: LocationGenerator, F: FileNameGenerator> {
inner: Option<RollingFileWriter<B, L, F>>,
pub struct DataFileWriter<B: FileWriterBuilder, F: FileNameGenerator> {
inner: Option<RollingFileWriter<B, F>>,
partition_key: Option<PartitionKey>,
}

#[async_trait::async_trait]
impl<B, L, F> IcebergWriter for DataFileWriter<B, L, F>
impl<B, F> IcebergWriter for DataFileWriter<B, F>
where
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
{
async fn write(&mut self, batch: RecordBatch) -> Result<()> {
Expand Down Expand Up @@ -112,10 +109,9 @@ where
}
}

impl<B, L, F> CurrentFileStatus for DataFileWriter<B, L, F>
impl<B, F> CurrentFileStatus for DataFileWriter<B, F>
where
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
{
fn current_file_path(&self) -> String {
Expand Down Expand Up @@ -180,7 +176,7 @@ mod test {
let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
pw,
file_io.clone(),
location_gen,
Arc::new(location_gen),
file_name_gen,
);

Expand Down Expand Up @@ -266,7 +262,7 @@ mod test {
let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
parquet_writer_builder,
file_io.clone(),
location_gen,
Arc::new(location_gen),
file_name_gen,
);

Expand Down
40 changes: 13 additions & 27 deletions crates/iceberg/src/writer/base_writer/equality_delete_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,33 +30,25 @@ use crate::arrow::schema_to_arrow_schema;
use crate::error::invalid_data;
use crate::spec::{DataFile, PartitionKey, Schema, SchemaRef};
use crate::writer::file_writer::FileWriterBuilder;
use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator};
use crate::writer::file_writer::location_generator::FileNameGenerator;
use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder};
use crate::writer::{IcebergWriter, IcebergWriterBuilder};
use crate::{Error, ErrorKind, Result};

/// Builder for `EqualityDeleteWriter`.
#[derive(Debug)]
pub struct EqualityDeleteFileWriterBuilder<
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
> {
inner: RollingFileWriterBuilder<B, L, F>,
pub struct EqualityDeleteFileWriterBuilder<B: FileWriterBuilder, F: FileNameGenerator> {
inner: RollingFileWriterBuilder<B, F>,
config: EqualityDeleteWriterConfig,
}

impl<B, L, F> EqualityDeleteFileWriterBuilder<B, L, F>
impl<B, F> EqualityDeleteFileWriterBuilder<B, F>
where
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
{
/// Create a new `EqualityDeleteFileWriterBuilder` using a `RollingFileWriterBuilder`.
pub fn new(
inner: RollingFileWriterBuilder<B, L, F>,
config: EqualityDeleteWriterConfig,
) -> Self {
pub fn new(inner: RollingFileWriterBuilder<B, F>, config: EqualityDeleteWriterConfig) -> Self {
Self { inner, config }
}
}
Expand Down Expand Up @@ -149,13 +141,12 @@ impl EqualityDeleteWriterConfig {
}

#[async_trait::async_trait]
impl<B, L, F> IcebergWriterBuilder for EqualityDeleteFileWriterBuilder<B, L, F>
impl<B, F> IcebergWriterBuilder for EqualityDeleteFileWriterBuilder<B, F>
where
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
{
type R = EqualityDeleteFileWriter<B, L, F>;
type R = EqualityDeleteFileWriter<B, F>;

async fn build(&self, partition_key: Option<PartitionKey>) -> Result<Self::R> {
Ok(EqualityDeleteFileWriter {
Expand All @@ -169,22 +160,17 @@ where

/// Writer used to write equality delete files.
#[derive(Debug)]
pub struct EqualityDeleteFileWriter<
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
> {
inner: Option<RollingFileWriter<B, L, F>>,
pub struct EqualityDeleteFileWriter<B: FileWriterBuilder, F: FileNameGenerator> {
inner: Option<RollingFileWriter<B, F>>,
projector: RecordBatchProjector,
equality_ids: Vec<i32>,
partition_key: Option<PartitionKey>,
}

#[async_trait::async_trait]
impl<B, L, F> IcebergWriter for EqualityDeleteFileWriter<B, L, F>
impl<B, F> IcebergWriter for EqualityDeleteFileWriter<B, F>
where
B: FileWriterBuilder,
L: LocationGenerator,
F: FileNameGenerator,
{
async fn write(&mut self, batch: RecordBatch) -> Result<()> {
Expand Down Expand Up @@ -464,7 +450,7 @@ mod test {
let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
pb,
file_io.clone(),
location_gen,
Arc::new(location_gen),
file_name_gen,
);
let mut equality_delete_writer =
Expand Down Expand Up @@ -685,7 +671,7 @@ mod test {
let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
pb,
file_io.clone(),
location_gen,
Arc::new(location_gen),
file_name_gen,
);
let mut equality_delete_writer =
Expand Down Expand Up @@ -814,7 +800,7 @@ mod test {
let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
pb,
file_io.clone(),
location_gen,
Arc::new(location_gen),
file_name_gen,
);
let mut equality_delete_writer =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -591,11 +591,7 @@ mod test {
/// configured with the [`position_delete_schema`].
fn writer_setup(
temp_dir: &TempDir,
) -> PositionDeleteFileWriterBuilder<
ParquetWriterBuilder,
DefaultLocationGenerator,
DefaultFileNameGenerator,
> {
) -> PositionDeleteFileWriterBuilder<ParquetWriterBuilder, DefaultFileNameGenerator> {
let file_io = FileIO::new_with_fs();
let location_gen = DefaultLocationGenerator::with_data_location(
temp_dir.path().to_str().unwrap().to_string(),
Expand All @@ -609,7 +605,7 @@ mod test {
let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
parquet_writer_builder,
file_io,
location_gen,
Arc::new(location_gen),
file_name_gen,
);
PositionDeleteFileWriterBuilder::new(rolling_writer_builder)
Expand Down
Loading
Loading