Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions crates/iceberg/public-api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3483,6 +3483,7 @@ pub fn iceberg::table::Table::encryption_manager(&self) -> core::option::Option<
pub fn iceberg::table::Table::file_io(&self) -> &iceberg::io::FileIO
pub fn iceberg::table::Table::identifier(&self) -> &iceberg::TableIdent
pub fn iceberg::table::Table::inspect(&self) -> iceberg::inspect::MetadataTable<'_>
pub fn iceberg::table::Table::location_generator(&self) -> iceberg::Result<iceberg::writer::file_writer::location_generator::TableLocationGenerator>
pub fn iceberg::table::Table::manifest_list_reader(&self, snapshot: &iceberg::spec::SnapshotRef) -> iceberg::spec::ManifestListReader
pub fn iceberg::table::Table::manifest_reader(&self) -> iceberg::spec::ManifestReader
pub fn iceberg::table::Table::metadata(&self) -> &iceberg::spec::TableMetadata
Expand Down Expand Up @@ -3691,6 +3692,13 @@ impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::Obje
pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
pub struct iceberg::writer::file_writer::location_generator::TableLocationGenerator
impl core::clone::Clone for iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::clone(&self) -> iceberg::writer::file_writer::location_generator::TableLocationGenerator
impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
pub trait iceberg::writer::file_writer::location_generator::FileNameGenerator: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static
pub fn iceberg::writer::file_writer::location_generator::FileNameGenerator::generate_file_name(&self) -> alloc::string::String
impl iceberg::writer::file_writer::location_generator::FileNameGenerator for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator
Expand All @@ -3701,6 +3709,8 @@ impl iceberg::writer::file_writer::location_generator::LocationGenerator for ice
pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::TableLocationGenerator
pub fn iceberg::writer::file_writer::location_generator::TableLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String
pub mod iceberg::writer::file_writer::rolling_writer
pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriter<B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator>
impl<B, L, F> iceberg::writer::file_writer::rolling_writer::RollingFileWriter<B, L, F> where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator
Expand Down
27 changes: 4 additions & 23 deletions crates/iceberg/src/cow_rewrite/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,7 @@ use crate::table::Table;
use crate::writer::IcebergWriterBuilder;
use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder;
use crate::writer::file_writer::ParquetWriterBuilder;
use crate::writer::file_writer::location_generator::{
DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator,
ObjectStorageLocationGenerator,
};
use crate::writer::file_writer::location_generator::DefaultFileNameGenerator;
use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder;

/// Builds a boxed replacement-data-file writer.
Expand All @@ -44,9 +41,8 @@ use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder;
/// batch is written, so it is safe to construct one optimistically and only
/// write to it once a source file is known to have changed.
///
/// The location layout follows `write.object-storage.enabled`: tables that opt
/// into the object-storage layout get hash-entropy paths from
/// [`ObjectStorageLocationGenerator`], everything else gets the default layout.
/// File locations follow the layout selected by the table properties; see
/// [`Table::location_generator`].
pub(crate) async fn build_replacement_writer(
table: &Table,
write_schema: SchemaRef,
Expand All @@ -72,27 +68,12 @@ pub(crate) async fn build_replacement_writer(
parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone());
}

if table_props.write_object_storage_enabled()? {
let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?;
build_data_file_writer(table, parquet_builder, location_generator, partition_key).await
} else {
let location_generator = DefaultLocationGenerator::new(table.metadata())?;
build_data_file_writer(table, parquet_builder, location_generator, partition_key).await
}
}

async fn build_data_file_writer<L: LocationGenerator>(
table: &Table,
parquet_builder: ParquetWriterBuilder,
location_generator: L,
partition_key: Option<PartitionKey>,
) -> Result<Box<dyn crate::writer::IcebergWriter>> {
let location_generator = table.location_generator()?;
let file_name_generator = DefaultFileNameGenerator::new(
format!("cow-rewrite-{}", Uuid::now_v7()),
None,
DataFileFormat::Parquet,
);
let table_props = table.metadata().table_properties();
let rolling_builder = RollingFileWriterBuilder::new(
parquet_builder,
table_props.write_target_file_size_bytes()?,
Expand Down
12 changes: 12 additions & 0 deletions crates/iceberg/src/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ use crate::scan::TableScanBuilder;
use crate::spec::{
ManifestListReader, ManifestReader, SchemaRef, SnapshotRef, TableMetadata, TableMetadataRef,
};
use crate::writer::file_writer::location_generator::TableLocationGenerator;
use crate::{Result, TableIdent};

/// Builder to create table scan.
Expand Down Expand Up @@ -325,6 +326,17 @@ impl Table {
pub fn reader_builder(&self) -> ArrowReaderBuilder {
ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone())
}

/// Creates the location generator for the layout selected by the table properties.
///
/// Java's `write.location-provider.impl` is not supported and is ignored.
///
/// # Errors
///
/// Returns an error if a table property that determines the layout has an invalid value.
pub fn location_generator(&self) -> Result<TableLocationGenerator> {
TableLocationGenerator::new(self.metadata())
}
}

/// `StaticTable` is a read-only table struct that can be created from a metadata file or from `TableMetaData` without a catalog.
Expand Down
140 changes: 140 additions & 0 deletions crates/iceberg/src/writer/file_writer/location_generator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ const ENTROPY_DIR_DEPTH: usize = 3;
#[derive(Clone, Debug)]
/// `DefaultLocationGenerator` used to generate the data dir location of data file.
/// The location is generated based on the table location and the data location in table properties.
///
/// Paths use `{data location}/{partition path}/{file name}`, omitting the
/// partition path for unpartitioned files. It ignores `write.object-storage.enabled`; use
/// [`Table::location_generator`](crate::table::Table::location_generator) for the layout the
/// table properties select.
pub struct DefaultLocationGenerator {
data_location: String,
}
Expand Down Expand Up @@ -203,6 +208,49 @@ impl LocationGenerator for ObjectStorageLocationGenerator {
}
}

/// Generates data file paths using the layout configured in table properties.
///
/// It uses [`ObjectStorageLocationGenerator`] when
/// [`write.object-storage.enabled`](crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED)
/// is `true`, and [`DefaultLocationGenerator`] otherwise. It is created by
/// [`Table::location_generator`](crate::table::Table::location_generator).
#[derive(Clone, Debug)]
pub struct TableLocationGenerator {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wrote up some context in #3729 on why I've gone with this approach but open to change / discussion if folks have thoughts. A new generator here lets us keep compatibility with current client behaviour, but the downside is that DefaultLocationGenerator reads naturally yet may not do what's desired.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IAlso, Java and Python support custom location providers so that is worth considering too w.r.t. the API design we introduce here. I think the proposed design should be able to support that by e.g. adding Custom(Arc<dyn <ErasedGenerator>) to the Layout enum)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think we should consider custom location provider here. enum seems like an acceptable workaround to me, but I don't want to block the custom location provider support for datafusion. Would appreciate more clarity here!

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(You're right, I'm having a think here on the best shape)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you - on custom providers, I think they'd come in through the catalog builder like storage factories with the catalog handing those to each Table, not TableMetadata. So I've moved the entry point to Table::location_generator().

To check, I POC-ed what custom providers might look like here on top of this API choice i.e. Table::location_generator() doesn't change meaning e.g. DataFusion picks it up with no changes. (The wiring is LocationGeneratorFactory passed in through CatalogBuilder::with_location_generator_factory)

Separately, I do wonder why location generators aren't trait objects like Storage - i.e. if we were building this from scratch, we could pass Arc<dyn LocationGenerator> around instead with this PR's Table::location_generator() returning that too. #3738 is what that might look like (this is with custom providers on top).

That's breaking so I've kept this PR additive for now, but I am curious for thoughts on that alternative design.

inner: Layout,
}

#[derive(Clone, Debug)]
enum Layout {
Default(DefaultLocationGenerator),
ObjectStorage(ObjectStorageLocationGenerator),
}

impl TableLocationGenerator {
/// Creates a generator from table metadata.
pub(crate) fn new(table_metadata: &TableMetadata) -> Result<Self> {
let inner = if table_metadata
.table_properties()
.write_object_storage_enabled()?
{
Layout::ObjectStorage(ObjectStorageLocationGenerator::new(table_metadata)?)
} else {
Layout::Default(DefaultLocationGenerator::new(table_metadata)?)
};
Ok(Self { inner })
}
}

impl LocationGenerator for TableLocationGenerator {
fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String {
match &self.inner {
Layout::Default(generator) => generator.generate_location(partition_key, file_name),
Layout::ObjectStorage(generator) => {
generator.generate_location(partition_key, file_name)
}
}
}
}

/// Derive the `{parent}/{name}` context from a table location, mirroring Hadoop's
/// `Path.getParent().getName()` / `Path.getName()`.
fn path_context(table_location: &str) -> String {
Expand Down Expand Up @@ -291,15 +339,18 @@ pub(crate) mod test {
use std::collections::HashMap;
use std::sync::Arc;

use rstest::rstest;
use uuid::Uuid;

use super::LocationGenerator;
use crate::ErrorKind;
use crate::spec::{
FormatVersion, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Schema,
Struct, StructType, TableMetadata, TableProperties, Transform, Type,
};
use crate::writer::file_writer::location_generator::{
DefaultLocationGenerator, FileNameGenerator, ObjectStorageLocationGenerator,
TableLocationGenerator,
};

#[test]
Expand Down Expand Up @@ -609,6 +660,95 @@ pub(crate) mod test {
);
}

#[rstest]
#[case::unset(&[], [
"s3://data.db/table/data/test.parquet",
"s3://data.db/table/data/data%231=val%231/test.parquet",
])]
#[case::disabled(&[(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, "false")], [
"s3://data.db/table/data/test.parquet",
"s3://data.db/table/data/data%231=val%231/test.parquet",
])]
#[case::java_location_provider(&[("write.location-provider.impl", "com.example.Provider")], [
"s3://data.db/table/data/test.parquet",
"s3://data.db/table/data/data%231=val%231/test.parquet",
])]
#[case::enabled(&[(TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, "true")], [
"s3://data.db/table/data/0110/1010/0011/11101000/test.parquet",
"s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet",
])]
fn test_table_location_follows_properties(
#[case] properties: &[(&str, &str)],
#[case] expected: [&str; 2],
) {
let metadata = table_metadata_with(
"s3://data.db/table",
properties
.iter()
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect(),
);
let schema = Arc::new(
Schema::builder()
.with_fields(vec![
NestedField::required(1, "data#1", Type::Primitive(PrimitiveType::String))
.into(),
])
.build()
.unwrap(),
);
let partition_spec = PartitionSpec::builder(schema.clone())
.add_partition_field("data#1", "data#1", Transform::Identity)
.unwrap()
.build()
.unwrap();
let partition_key = PartitionKey::new(
partition_spec,
schema,
Struct::from_iter([Some(Literal::string("val#1"))]),
);
let generator = TableLocationGenerator::new(&metadata).unwrap();

assert_eq!(
[
generator.generate_location(None, "test.parquet"),
generator.generate_location(Some(&partition_key), "test.parquet"),
],
expected
);
}

#[test]
fn test_table_location_rejects_invalid_property() {
let metadata = table_metadata_with(
"s3://data.db/table",
HashMap::from([(
TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED.to_string(),
"yes".to_string(),
)]),
);
let err = TableLocationGenerator::new(&metadata).unwrap_err();

assert_eq!(err.kind(), ErrorKind::DataInvalid);
}

#[test]
fn test_default_location_ignores_object_storage_property() {
let metadata = table_metadata_with(
"s3://data.db/table",
HashMap::from([(
TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED.to_string(),
"true".to_string(),
)]),
);
let generator = DefaultLocationGenerator::new(&metadata).unwrap();

assert_eq!(
generator.generate_location(None, "test.parquet"),
"s3://data.db/table/data/test.parquet"
);
}

/// Build a minimal `TableMetadata` for location generator tests.
fn table_metadata_with(location: &str, properties: HashMap<String, String>) -> TableMetadata {
TableMetadata {
Expand Down
24 changes: 8 additions & 16 deletions crates/iceberg/src/writer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,7 @@
//! use iceberg::transaction::Transaction;
//! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
//! use iceberg::writer::file_writer::ParquetWriterBuilder;
//! use iceberg::writer::file_writer::location_generator::{
//! DefaultFileNameGenerator, DefaultLocationGenerator,
//! };
//! use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator;
//! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
//! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent};
//! use parquet::file::properties::WriterProperties;
Expand All @@ -78,7 +76,7 @@
//! let table = catalog
//! .load_table(&TableIdent::from_strs(["hello", "world"])?)
//! .await?;
//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap();
//! let location_generator = table.location_generator()?;
//! let file_name_generator = DefaultFileNameGenerator::new(
//! "test".to_string(),
//! None,
Expand Down Expand Up @@ -124,9 +122,7 @@
//! use iceberg::spec::{DataFile, PartitionKey};
//! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
//! use iceberg::writer::file_writer::ParquetWriterBuilder;
//! use iceberg::writer::file_writer::location_generator::{
//! DefaultFileNameGenerator, DefaultLocationGenerator,
//! };
//! use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator;
//! use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
//! use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, Result, TableIdent};
//! use parquet::file::properties::WriterProperties;
Expand Down Expand Up @@ -207,7 +203,7 @@
//! table.metadata().current_schema().clone(),
//! Struct::from_iter(vec![Some(Literal::string("Seattle"))]),
//! );
//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap();
//! let location_generator = table.location_generator()?;
//! let file_name_generator = DefaultFileNameGenerator::new(
//! "test".to_string(),
//! None,
Expand Down Expand Up @@ -261,9 +257,7 @@
//! # use iceberg::{Catalog, CatalogBuilder, Result, TableIdent};
//! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
//! # use iceberg::writer::file_writer::ParquetWriterBuilder;
//! # use iceberg::writer::file_writer::location_generator::{
//! # DefaultFileNameGenerator, DefaultLocationGenerator,
//! # };
//! # use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator;
//! # use parquet::file::properties::WriterProperties;
//! # use std::collections::HashMap;
//! # #[tokio::main]
Expand All @@ -272,7 +266,7 @@
//! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())]))
//! # .await?;
//! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?;
//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap();
//! # let location_generator = table.location_generator()?;
//! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet);
//! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone());
//! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
Expand Down Expand Up @@ -325,9 +319,7 @@
//! # use iceberg::{Catalog, CatalogBuilder, Result, TableIdent};
//! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
//! # use iceberg::writer::file_writer::ParquetWriterBuilder;
//! # use iceberg::writer::file_writer::location_generator::{
//! # DefaultFileNameGenerator, DefaultLocationGenerator,
//! # };
//! # use iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator;
//! # use parquet::file::properties::WriterProperties;
//! # use std::collections::HashMap;
//! # #[tokio::main]
Expand All @@ -336,7 +328,7 @@
//! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())]))
//! # .await?;
//! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?;
//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap();
//! # let location_generator = table.location_generator()?;
//! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet);
//! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone());
//! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size(
Expand Down
Loading