diff --git a/Cargo.lock b/Cargo.lock index 5d72ad4fb..cb00f04e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3612,6 +3612,27 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777d18aa0f12f8b285331cd867133ee14422b3f023f6d388034c47d43e28786a" +dependencies = [ + "approx", + "num-traits", + "serde", + "thiserror 2.0.20", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -4043,7 +4064,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=7fda030197a61057422b12ca4cf2cc783a5607f0#7fda030197a61057422b12ca4cf2cc783a5607f0" dependencies = [ "aes-gcm", "anyhow", @@ -4051,6 +4072,7 @@ dependencies = [ "array-init", "arrow-arith", "arrow-array", + "arrow-avro", "arrow-buffer", "arrow-cast", "arrow-ord", @@ -4069,6 +4091,7 @@ dependencies = [ "fastnum", "flate2", "fnv", + "foldhash 0.2.0", "form_urlencoded", "futures", "iceberg-property-macro", @@ -4078,6 +4101,7 @@ dependencies = [ "once_cell", "ordered-float", "parquet", + "parquet-geospatial", "rand 0.9.5", "reqwest 0.12.28", "roaring", @@ -4101,7 +4125,7 @@ dependencies = [ [[package]] name = "iceberg-datafusion" version = "0.12.0" -source = "git+https://github.com/apache/datafusion-iceberg.git?rev=20327db58c7a44330366da00a34a26e866d0da4d#20327db58c7a44330366da00a34a26e866d0da4d" +source = "git+https://github.com/DataDog/datafusion-iceberg?rev=4b4ec3cba8f0a12f848661bffd1c224badbb28b3#4b4ec3cba8f0a12f848661bffd1c224badbb28b3" dependencies = [ "async-trait", "bytes", @@ -4119,7 +4143,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=7fda030197a61057422b12ca4cf2cc783a5607f0#7fda030197a61057422b12ca4cf2cc783a5607f0" dependencies = [ "proc-macro2", "quote", @@ -4129,7 +4153,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +source = "git+https://github.com/DataDog/iceberg-rust?rev=7fda030197a61057422b12ca4cf2cc783a5607f0#7fda030197a61057422b12ca4cf2cc783a5607f0" dependencies = [ "anyhow", "async-trait", @@ -5118,6 +5142,28 @@ dependencies = [ "libm", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num_threads" version = "0.1.7" @@ -5567,6 +5613,7 @@ dependencies = [ "num-integer", "num-traits", "object_store", + "parquet-geospatial", "ring", "seq-macro", "simdutf8", @@ -5576,6 +5623,19 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "parquet-geospatial" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634827365851180ca26bf54c76e970dc1c14c075684c0580ee97d8d0c953faf1" +dependencies = [ + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "wkb", +] + [[package]] name = "pbkdf2" version = "0.12.2" @@ -5912,6 +5972,15 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -7821,6 +7890,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.19+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd43688c4858d7264ae1cf9a9f874a67c3feb74a883b1cd7bebd6a4ea8307218" +dependencies = [ + "indexmap 2.14.2", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "tonic" version = "0.14.6" @@ -8642,12 +8741,33 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wkb" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04ab2f2845abe967a213998446345a1a6fc9a8f318cd88f668a8e507861d13bc" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "writeable" version = "0.6.4" diff --git a/datafusion/distributed-iceberg-test-tools/Cargo.toml b/datafusion/distributed-iceberg-test-tools/Cargo.toml index c39d7ae69..6fe58122e 100644 --- a/datafusion/distributed-iceberg-test-tools/Cargo.toml +++ b/datafusion/distributed-iceberg-test-tools/Cargo.toml @@ -15,7 +15,7 @@ datafusion = { workspace = true, features = ["parquet", "sql"] } datafusion-distributed = { path = "../distributed", features = ["integration"] } datafusion-proto = { workspace = true } # Match datafusion-distributed-iceberg's iceberg-rust revision so tables share one type. -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "7fda030197a61057422b12ca4cf2cc783a5607f0" } rand = { version = "0.10", default-features = false, features = ["alloc"] } rand_pcg = "0.10" tempfile = "3.18" diff --git a/datafusion/distributed-iceberg/Cargo.toml b/datafusion/distributed-iceberg/Cargo.toml index 31e11fe37..7e941aff2 100644 --- a/datafusion/distributed-iceberg/Cargo.toml +++ b/datafusion/distributed-iceberg/Cargo.toml @@ -13,10 +13,10 @@ bytes = "1" datafusion = { workspace = true, features = ["sql"] } datafusion-proto = { workspace = true } # Match datafusion-iceberg's iceberg-rust revision so FileScanTask has one shared type. -iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg.git", rev = "20327db58c7a44330366da00a34a26e866d0da4d" } +iceberg-datafusion = { git = "https://github.com/DataDog/datafusion-iceberg", rev = "4b4ec3cba8f0a12f848661bffd1c224badbb28b3" } futures = "0.3" -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } -iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } +iceberg = { git = "https://github.com/DataDog/iceberg-rust", rev = "7fda030197a61057422b12ca4cf2cc783a5607f0" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust", rev = "7fda030197a61057422b12ca4cf2cc783a5607f0", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } datafusion-distributed = { path = "../distributed", features = ["sql"] } delegate = "0.13" prost = "0.14.1"