From 746ff59abc3c7dfbda257fe4a5148da5696e1841 Mon Sep 17 00:00:00 2001 From: Yi Li Date: Tue, 22 Sep 2026 11:35:15 -0700 Subject: [PATCH 1/6] [Spark] Add Spark + Iceberg REST catalog integration tests - Add `IcebergTableReadWriteTest` (+ managed/external subclasses) driving Iceberg tables through Iceberg's own REST `SparkCatalog` against UC's Iceberg REST catalog, on a local `file://` warehouse; reuses `BaseTableReadWriteTest`'s create/read/write matrix. - Add `DeltaIcebergCrossFormatJoinTest`: one session wired with both the UC connector (Delta) and the Iceberg REST catalog, joining a Delta table against an Iceberg table. - Refactor `BaseSparkIntegrationTest` so all suites share `createSparkSessionWithCatalogs`: - per-catalog `isIcebergCatalog` switch selects UC-connector vs Iceberg wiring; session extensions are derived from the catalog mix - extract `ucSingleCatalogConfig` / `icebergCatalogConfig` / `catalogConfKey` - Make the shared `BaseTableReadWriteTest` tests Iceberg-aware via `testingIceberg()`, so `testTableOperations`, `testListTablesPagination`, and `testHyphenInTableName` run for Iceberg instead of being reimplemented; the non-Iceberg path is unchanged. - `build.sbt`: add `iceberg-spark-runtime` (Spark 4.0/4.1 only) as a test dep and open `java.base/java.nio` in `Test / javaOptions` for Arrow reads. Signed-off-by: Yi Li --- build.sbt | 14 +- .../spark/BaseSparkIntegrationTest.java | 105 +++++++-- .../spark/BaseTableReadWriteTest.java | 117 ++++++---- .../spark/UCTablePropertiesTest.java | 3 +- .../DeltaIcebergCrossFormatJoinTest.java | 88 ++++++++ .../IcebergExternalTableReadWriteTest.java | 37 ++++ .../IcebergManagedTableReadWriteTest.java | 20 ++ .../spark/IcebergTableReadWriteTest.java | 203 ++++++++++++++++++ 8 files changed, 529 insertions(+), 58 deletions(-) create mode 100644 connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java create mode 100644 connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergExternalTableReadWriteTest.java create mode 100644 connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergManagedTableReadWriteTest.java create mode 100644 connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java diff --git a/build.sbt b/build.sbt index ac5e082f88..7d728085c9 100644 --- a/build.sbt +++ b/build.sbt @@ -35,6 +35,16 @@ def deltaSparkTestDeps: Seq[ModuleID] = if (sys.props.getOrElse("skipDeltaSpark", "false").toBoolean) Seq.empty else Seq("io.delta" %% s"delta-spark_$sparkMajorMinorVersion" % deltaVersion % Test) +// iceberg-spark-runtime is published for Spark 4.0 and 4.1 (Iceberg 1.11.0) but not Spark 4.2, so +// it is added only for the versions that have a runtime. The Iceberg-on-Spark integration test +// source lives under src/test/scala-shims/spark-4.0-4.1, which is compiled only for those +// versions, so nothing references this dep on Spark 4.2. +def icebergSparkTestDeps: Seq[ModuleID] = + if (CrossSparkVersions.getSparkVersionSpec().isAtLeast(4, 2)) Seq.empty + else + Seq( + "org.apache.iceberg" % s"iceberg-spark-runtime-${sparkMajorMinorVersion}_2.13" % icebergVersion % Test) + // Apache Snapshots resolver is in build/sbt-config/repositories (global). // No per-module sparkResolvers needed. lazy val hadoopVersion = sys.props.getOrElse("hadoopVersion", "3.4.2") @@ -695,6 +705,8 @@ lazy val spark = (project in file("connectors/spark")) "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", ), + // Arrow-backed vectorized reads need java.nio opened for off-heap memory access. + Test / javaOptions += "--add-opens=java.base/java.nio=ALL-UNNAMED", javafmtCheckSettings(), javaCheckstyleSettings("dev/checkstyle-config.xml"), Compile / compile / javacOptions ++= javacRelease11, @@ -743,7 +755,7 @@ lazy val spark = (project in file("connectors/spark")) "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % Test, "org.projectlombok" % "lombok" % "1.18.32" % Test, "com.google.cloud.bigdataoss" % "gcs-connector" % "3.0.2" % Test classifier "shaded", - ) ++ deltaSparkTestDeps, + ) ++ deltaSparkTestDeps ++ icebergSparkTestDeps, dependencyOverrides ++= Seq( "com.fasterxml.jackson.core" % "jackson-databind" % sparkJacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkJacksonVersion, diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java index 5023854cd6..a1e9574754 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java @@ -32,6 +32,9 @@ public abstract class BaseSparkIntegrationTest extends BaseCRUDTest { protected ArrayList createdCatalogs = new ArrayList<>(); protected static final String SPARK_CATALOG = "spark_catalog"; + /** The server's Iceberg REST catalog path, served under the UC base path. */ + private static final String ICEBERG_REST_PATH = "/api/2.1/unity-catalog/iceberg"; + /** S3 bucket with no credentials configured on server - for testing SSP fallback. */ public static final String NO_CREDS_BUCKET = "test-bucket-2-no-creds"; @@ -39,6 +42,17 @@ public abstract class BaseSparkIntegrationTest extends BaseCRUDTest { // Each test would create this session. It will be closed automatically. protected SparkSession session; + /** + * True when this suite drives the given catalog as Iceberg's own REST {@code SparkCatalog} + * against the server's Iceberg REST catalog, rather than through the UC Spark connector (which + * has no Iceberg support). Per catalog, so one session can mix both (e.g. a Delta join against + * Iceberg); subclasses flip this, and {@link #createSparkSessionWithCatalogs} is the only + * consumer, so no subclass needs to re-implement session creation. + */ + protected boolean isIcebergCatalog(String catalog) { + return false; + } + private void createCommonResources() throws ApiException { // Common setup operations such as creating a catalog and schema catalogOperations.createCatalog( @@ -52,24 +66,45 @@ protected SparkSession createSparkSessionWithCatalogs(String... catalogs) { return createSparkSessionWithCatalogs(true, true, catalogs); } + /** The base Spark builder shared by all integration tests (local master, small shuffle width). */ + protected SparkSession.Builder newSparkSessionBuilder() { + return SparkSession.builder() + .appName("test") + .master("local[*]") + .config("spark.sql.shuffle.partitions", "4"); + } + protected SparkSession createSparkSessionWithCatalogs( boolean renewCred, boolean credScopedFsEnabled, String... catalogs) { + // renewCred / credScopedFsEnabled are UC-connector options with no Iceberg REST analog. + // A catalog is wired either as an Iceberg REST catalog or through the UC connector + // (UCSingleCatalog), which serves Delta, Parquet, and the other non-Iceberg formats. + boolean anyIceberg = false; + boolean anyUcConnector = false; + for (String catalog : catalogs) { + if (isIcebergCatalog(catalog)) { + anyIceberg = true; + } else { + anyUcConnector = true; + } + } + // Both extensions can coexist in one session, which is what lets a single query join a Delta + // table against an Iceberg table. The UC connector carries Delta support, so its path loads the + // Delta extension (harmless for the non-Delta formats it also serves, e.g. Parquet). + List extensions = new ArrayList<>(); + if (anyUcConnector) { + extensions.add("io.delta.sql.DeltaSparkSessionExtension"); + } + if (anyIceberg) { + extensions.add("org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"); + } SparkSession.Builder builder = - SparkSession.builder() - .appName("test") - .master("local[*]") - .config("spark.sql.shuffle.partitions", "4") - .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension"); + newSparkSessionBuilder().config("spark.sql.extensions", String.join(",", extensions)); for (String catalog : catalogs) { - String catalogConf = "spark.sql.catalog." + catalog; builder = - builder - .config(catalogConf, UCSingleCatalog.class.getName()) - .config(catalogConf + "." + OptionsUtil.URI, serverConfig.getServerUrl()) - .config(catalogConf + "." + OptionsUtil.TOKEN, serverConfig.getAuthToken()) - .config(catalogConf + "." + OptionsUtil.WAREHOUSE, catalog) - .config(catalogConf + "." + OptionsUtil.RENEW_CREDENTIAL_ENABLED, renewCred) - .config(catalogConf + "." + OptionsUtil.CRED_SCOPED_FS_ENABLED, credScopedFsEnabled); + isIcebergCatalog(catalog) + ? icebergCatalogConfig(builder, catalog) + : ucSingleCatalogConfig(builder, catalog, renewCred, credScopedFsEnabled); if (!List.of(SPARK_CATALOG, CATALOG_NAME).contains(catalog)) { createTestCatalog(catalog); } @@ -78,9 +113,12 @@ protected SparkSession createSparkSessionWithCatalogs( // (DeltaLog.checkRequiredConfigurations); otherwise a Delta table -- even in a named UC catalog // -- fails with DELTA_CONFIGURE_SPARK_SESSION_WITH_EXTENSION_AND_CATALOG. When the caller did // not configure spark_catalog above, point it at Delta's own catalog to satisfy that check. - if (!List.of(catalogs).contains(SPARK_CATALOG)) { + // This applies to any UC-connector session (the connector may serve Delta); Iceberg has no such + // requirement and these suites address tables only through the named UC catalog, so a + // pure-Iceberg session needs no spark_catalog override and leaves it as the built-in one. + if (anyUcConnector && !List.of(catalogs).contains(SPARK_CATALOG)) { builder.config( - "spark.sql.catalog." + SPARK_CATALOG, "org.apache.spark.sql.delta.catalog.DeltaCatalog"); + catalogConfKey(SPARK_CATALOG), "org.apache.spark.sql.delta.catalog.DeltaCatalog"); } // Use fake file system for cloud storage so that we can test credentials. builder.config("spark.hadoop.fs.s3.impl", S3CredentialTestFileSystem.class.getName()); @@ -89,6 +127,43 @@ protected SparkSession createSparkSessionWithCatalogs( return builder.getOrCreate(); } + /** The Spark SQL config key prefix for a catalog, e.g. {@code spark.sql.catalog.}. */ + private static String catalogConfKey(String catalog) { + return "spark.sql.catalog." + catalog; + } + + /** Wires one catalog through the UC Spark connector ({@link UCSingleCatalog}), the Delta path. */ + private SparkSession.Builder ucSingleCatalogConfig( + SparkSession.Builder builder, + String catalog, + boolean renewCred, + boolean credScopedFsEnabled) { + String catalogConf = catalogConfKey(catalog); + return builder + .config(catalogConf, UCSingleCatalog.class.getName()) + .config(catalogConf + "." + OptionsUtil.URI, serverConfig.getServerUrl()) + .config(catalogConf + "." + OptionsUtil.TOKEN, serverConfig.getAuthToken()) + .config(catalogConf + "." + OptionsUtil.WAREHOUSE, catalog) + .config(catalogConf + "." + OptionsUtil.RENEW_CREDENTIAL_ENABLED, renewCred) + .config(catalogConf + "." + OptionsUtil.CRED_SCOPED_FS_ENABLED, credScopedFsEnabled); + } + + /** + * Wires one catalog as Iceberg's REST {@code SparkCatalog} against the server's Iceberg REST + * catalog, reading and writing data files through Iceberg's {@code HadoopFileIO} on the local + * warehouse. + */ + private SparkSession.Builder icebergCatalogConfig(SparkSession.Builder builder, String catalog) { + String catalogConf = catalogConfKey(catalog); + return builder + .config(catalogConf, "org.apache.iceberg.spark.SparkCatalog") + .config(catalogConf + ".type", "rest") + .config(catalogConf + ".uri", serverConfig.getServerUrl() + ICEBERG_REST_PATH) + .config(catalogConf + ".warehouse", catalog) + .config(catalogConf + ".io-impl", "org.apache.iceberg.hadoop.HadoopFileIO") + .config(catalogConf + ".token", serverConfig.getAuthToken()); + } + protected List sql(String statement, Object... args) { return session.sql(String.format(statement, args)).collectAsList(); } diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java index 766f582fb3..7fd80739a3 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java @@ -243,6 +243,10 @@ protected final boolean testingDelta() { return tableFormat().equalsIgnoreCase("DELTA"); } + protected final boolean testingIceberg() { + return tableFormat().equalsIgnoreCase("ICEBERG"); + } + protected abstract boolean isManagedTable(); protected final boolean canUpdateColumnsToUC() { @@ -434,12 +438,15 @@ public void testTimeTravelDeltaTable() { @MethodSource("cloudParameters") public void testTableOperations( String scheme, boolean renewCredEnabled, boolean credScopedFsEnabled) { - session = - createSparkSessionWithCatalogs( - renewCredEnabled, credScopedFsEnabled, SPARK_CATALOG, CATALOG_NAME); + // The UC connector drives both spark_catalog and a named catalog; Iceberg wires only the named + // catalog (spark_catalog stays Spark's built-in session catalog), so it puts both tables there. + String[] catalogs = + testingIceberg() ? new String[] {CATALOG_NAME} : new String[] {SPARK_CATALOG, CATALOG_NAME}; + String firstCatalog = testingIceberg() ? CATALOG_NAME : SPARK_CATALOG; + session = createSparkSessionWithCatalogs(renewCredEnabled, credScopedFsEnabled, catalogs); // t1 has (1, 'a') - String t1 = setupTable(scheme, SPARK_CATALOG, TEST_TABLE); + String t1 = setupTable(scheme, firstCatalog, TEST_TABLE); testTableReadWrite(t1); // t2 has (2, 'a') @@ -464,18 +471,35 @@ public void testTableOperations( } // Test SHOW TABLES - List tables1 = sql("SHOW TABLES in %s.%s", SPARK_CATALOG, SCHEMA_NAME); - assertThat(tables1).hasSize(1); - assertThat(tables1.get(0).getString(0)).isEqualTo(SCHEMA_NAME); - assertThat(tables1.get(0).getString(1)).isEqualTo(TEST_TABLE); - List tables2 = sql("SHOW TABLES in %s.%s", CATALOG_NAME, SCHEMA_NAME); - assertThat(tables2).hasSize(1); - assertThat(tables2.get(0).getString(0)).isEqualTo(SCHEMA_NAME); - assertThat(tables2.get(0).getString(1)).isEqualTo(ANOTHER_TEST_TABLE); - - assertThatThrownBy(() -> sql("SHOW TABLES in a.b.c")) - .isInstanceOf(ApiException.class) - .hasMessageContaining("Nested namespaces are not supported"); + if (testingIceberg()) { + // Both tables live in the single wired catalog. + List tableNames = + sql("SHOW TABLES in %s.%s", CATALOG_NAME, SCHEMA_NAME).stream() + .map(row -> row.getString(1)) + .collect(Collectors.toList()); + assertThat(tableNames).containsExactlyInAnyOrder(TEST_TABLE, ANOTHER_TEST_TABLE); + // A nested (multi-level) namespace is rejected by the Iceberg REST catalog. Asserted by class + // name, not an imported type, so this shared base still compiles on Spark 4.2 (no Iceberg on + // its classpath). + assertThatThrownBy(() -> sql("SHOW TABLES in %s.a.b", CATALOG_NAME)) + .satisfies( + t -> + assertThat(t.getClass().getName()) + .isEqualTo("org.apache.iceberg.exceptions.RESTException")); + } else { + List tables1 = sql("SHOW TABLES in %s.%s", SPARK_CATALOG, SCHEMA_NAME); + assertThat(tables1).hasSize(1); + assertThat(tables1.get(0).getString(0)).isEqualTo(SCHEMA_NAME); + assertThat(tables1.get(0).getString(1)).isEqualTo(TEST_TABLE); + List tables2 = sql("SHOW TABLES in %s.%s", CATALOG_NAME, SCHEMA_NAME); + assertThat(tables2).hasSize(1); + assertThat(tables2.get(0).getString(0)).isEqualTo(SCHEMA_NAME); + assertThat(tables2.get(0).getString(1)).isEqualTo(ANOTHER_TEST_TABLE); + + assertThatThrownBy(() -> sql("SHOW TABLES in a.b.c")) + .isInstanceOf(ApiException.class) + .hasMessageContaining("Nested namespaces are not supported"); + } // DROP TABLE for (String fullTableName : List.of(t1, t2)) { @@ -483,35 +507,45 @@ public void testTableOperations( sql("DROP TABLE %s", fullTableName); assertThat(session.catalog().tableExists(fullTableName)).isFalse(); } - // Delta < 4.3.0 forwards the 4-part identifier to UC, which rejects it server-side with - // ApiException("Nested namespaces are not supported"). Delta >= 4.3.0 ships - // UCDeltaCatalogClientImpl, which rejects the same input client-side with - // IllegalArgumentException("UC table identifier must be one of ...") before reaching UC. - boolean rejectedClientSide = - DeltaVersionUtils.isDeltaAtLeast(MIN_DELTA_VERSION_FOR_UC_DELTA_API); - Class expectedType = - rejectedClientSide ? IllegalArgumentException.class : ApiException.class; - String expectedMessage = - rejectedClientSide - ? "UC table identifier must be one of" - : "Nested namespaces are not supported"; - assertThatThrownBy(() -> sql("DROP TABLE a.b.c.d")) - .isInstanceOf(expectedType) - .hasMessageContaining(expectedMessage); + // Dropping a malformed multi-level table identifier is rejected. + if (testingIceberg()) { + assertThatThrownBy(() -> sql("DROP TABLE %s.a.b.c", CATALOG_NAME)) + .satisfies( + t -> + assertThat(t.getClass().getName()) + .isEqualTo("org.apache.iceberg.exceptions.RESTException")); + } else { + // Delta < 4.3.0 forwards the 4-part identifier to UC, which rejects it server-side with + // ApiException("Nested namespaces are not supported"). Delta >= 4.3.0 ships + // UCDeltaCatalogClientImpl, which rejects the same input client-side with + // IllegalArgumentException("UC table identifier must be one of ...") before reaching UC. + boolean rejectedClientSide = + DeltaVersionUtils.isDeltaAtLeast(MIN_DELTA_VERSION_FOR_UC_DELTA_API); + Class expectedType = + rejectedClientSide ? IllegalArgumentException.class : ApiException.class; + String expectedMessage = + rejectedClientSide + ? "UC table identifier must be one of" + : "Nested namespaces are not supported"; + assertThatThrownBy(() -> sql("DROP TABLE a.b.c.d")) + .isInstanceOf(expectedType) + .hasMessageContaining(expectedMessage); + } } // With page size 2 and 3 tables, pagination must occur (2 API calls) for all 3 to be returned. // Asserting all 3 names proves pagination worked end-to-end. @Test public void testListTablesPagination() { - session = createSparkSessionWithCatalogs(SPARK_CATALOG); + String catalog = testingIceberg() ? CATALOG_NAME : SPARK_CATALOG; + session = createSparkSessionWithCatalogs(catalog); Integer originalPageSize = PagedListingHelper.DEFAULT_PAGE_SIZE; try { PagedListingHelper.DEFAULT_PAGE_SIZE = 2; - setupTable(SPARK_CATALOG, "pagination_table_1"); - setupTable(SPARK_CATALOG, "pagination_table_2"); - setupTable(SPARK_CATALOG, "pagination_table_3"); - List tables = sql("SHOW TABLES in %s.%s", SPARK_CATALOG, SCHEMA_NAME); + setupTable(catalog, "pagination_table_1"); + setupTable(catalog, "pagination_table_2"); + setupTable(catalog, "pagination_table_3"); + List tables = sql("SHOW TABLES in %s.%s", catalog, SCHEMA_NAME); assertThat(tables).hasSize(3); List tableNames = tables.stream().map(row -> row.getString(1)).sorted().collect(Collectors.toList()); @@ -519,9 +553,9 @@ public void testListTablesPagination() { .containsExactly("pagination_table_1", "pagination_table_2", "pagination_table_3"); } finally { PagedListingHelper.DEFAULT_PAGE_SIZE = originalPageSize; - sql("DROP TABLE IF EXISTS %s.%s.pagination_table_1", SPARK_CATALOG, SCHEMA_NAME); - sql("DROP TABLE IF EXISTS %s.%s.pagination_table_2", SPARK_CATALOG, SCHEMA_NAME); - sql("DROP TABLE IF EXISTS %s.%s.pagination_table_3", SPARK_CATALOG, SCHEMA_NAME); + sql("DROP TABLE IF EXISTS %s.%s.pagination_table_1", catalog, SCHEMA_NAME); + sql("DROP TABLE IF EXISTS %s.%s.pagination_table_2", catalog, SCHEMA_NAME); + sql("DROP TABLE IF EXISTS %s.%s.pagination_table_3", catalog, SCHEMA_NAME); } } @@ -534,7 +568,10 @@ public void testHyphenInTableName() { String catalogName = "test-catalog-name"; String schemaName = "test-schema-name"; String tableName = "test-table-name"; - session = createSparkSessionWithCatalogs(SPARK_CATALOG, catalogName); + session = + testingIceberg() + ? createSparkSessionWithCatalogs(catalogName) + : createSparkSessionWithCatalogs(SPARK_CATALOG, catalogName); sql("CREATE SCHEMA `%s`.`%s`", catalogName, schemaName); String fullTableName = setupTable( diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/UCTablePropertiesTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/UCTablePropertiesTest.java index 7d01920405..2608aec1b4 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/UCTablePropertiesTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/UCTablePropertiesTest.java @@ -52,8 +52,7 @@ public void shouldPersistProperty_dropsReservedCredentialsAndSchemaKeys() { UCTableProperties.shouldPersistProperty( TableCatalog.OPTION_PREFIX + "fs.s3a.session.token")) .isFalse(); - assertThat( - UCTableProperties.shouldPersistProperty("spark.sql.sources.schema.part.0")) + assertThat(UCTableProperties.shouldPersistProperty("spark.sql.sources.schema.part.0")) .isFalse(); assertThat( UCTableProperties.shouldPersistProperty( diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java new file mode 100644 index 0000000000..934d38cae1 --- /dev/null +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java @@ -0,0 +1,88 @@ +package io.unitycatalog.spark; + +import static io.unitycatalog.server.utils.TestUtils.CATALOG_NAME; +import static io.unitycatalog.server.utils.TestUtils.SCHEMA_NAME; +import static io.unitycatalog.server.utils.TestUtils.createApiClient; +import static org.assertj.core.api.Assertions.assertThat; + +import io.unitycatalog.client.ApiException; +import io.unitycatalog.client.model.DataSourceFormat; +import io.unitycatalog.server.base.table.TableOperations; +import io.unitycatalog.server.sdk.tables.SdkTableOperations; +import io.unitycatalog.server.utils.ServerProperties; +import java.util.List; +import org.apache.spark.sql.Row; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Verifies that a single SparkSession can be configured with both a Delta-capable catalog (the UC + * connector, {@link UCSingleCatalog}) and an Iceberg catalog (Iceberg's own REST {@code + * SparkCatalog} against UC's Iceberg REST endpoint) at the same time, and can join a Delta table + * against an Iceberg table across the two catalogs. + * + *

Both catalogs point at the same UC server: the Delta table lives in the {@code spark_catalog} + * UC catalog (reached through the connector, which also satisfies Delta's required-catalog check) + * and the Iceberg table in {@code CATALOG_NAME} (reached through the Iceberg REST client). The + * shared {@code createSparkSessionWithCatalogs} derives both session extensions from the catalog + * mix, and storage is a local {@code file://} warehouse. + * + *

Compiled only for Spark 4.0 / 4.1: Iceberg 1.11.0 publishes no Spark 4.2 runtime, so this + * source lives under {@code src/test/scala-shims/spark-4.0-4.1} (see {@link + * IcebergTableReadWriteTest}). + */ +public class DeltaIcebergCrossFormatJoinTest extends BaseSparkIntegrationTest { + + private TableOperations tableOperations; + + @BeforeEach + @Override + public void setUp() { + super.setUp(); + tableOperations = new SdkTableOperations(createApiClient(serverConfig)); + } + + @Override + protected void setUpProperties() { + super.setUpProperties(); + serverProperties.setProperty(ServerProperties.Property.ICEBERG_TABLE_ENABLED.getKey(), "true"); + } + + // Only CATALOG_NAME is Iceberg; SPARK_CATALOG stays on the UC connector for Delta, so the base + // wires a mixed session with both extensions. + @Override + protected boolean isIcebergCatalog(String catalog) { + return catalog.equals(CATALOG_NAME); + } + + @Test + public void testJoinDeltaAndIcebergTables() throws ApiException { + session = createSparkSessionWithCatalogs(false, false, SPARK_CATALOG, CATALOG_NAME); + + String deltaTable = String.join(".", SPARK_CATALOG, SCHEMA_NAME, "delta_events"); + String icebergTable = String.join(".", CATALOG_NAME, SCHEMA_NAME, "iceberg_events"); + + sql("CREATE TABLE %s (id INT, delta_val STRING) USING delta", deltaTable); + sql("INSERT INTO %s VALUES (1, 'delta-one'), (2, 'delta-two')", deltaTable); + + sql("CREATE TABLE %s (id INT, iceberg_val STRING) USING iceberg", icebergTable); + sql("INSERT INTO %s VALUES (1, 'iceberg-one'), (3, 'iceberg-three')", icebergTable); + + // The two tables are genuinely different formats in UC. + assertThat(tableOperations.getTable(deltaTable).getDataSourceFormat()) + .isEqualTo(DataSourceFormat.DELTA); + assertThat(tableOperations.getTable(icebergTable).getDataSourceFormat()) + .isEqualTo(DataSourceFormat.ICEBERG); + + // Cross-format inner join on id: only id = 1 exists in both tables. + List joined = + sql( + "SELECT d.id, d.delta_val, i.iceberg_val FROM %s d JOIN %s i ON d.id = i.id", + deltaTable, icebergTable); + assertThat(joined).hasSize(1); + Row row = joined.get(0); + assertThat(row.getInt(0)).isEqualTo(1); + assertThat(row.getString(1)).isEqualTo("delta-one"); + assertThat(row.getString(2)).isEqualTo("iceberg-one"); + } +} diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergExternalTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergExternalTableReadWriteTest.java new file mode 100644 index 0000000000..b48b71a67c --- /dev/null +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergExternalTableReadWriteTest.java @@ -0,0 +1,37 @@ +package io.unitycatalog.spark; + +import java.io.File; +import lombok.SneakyThrows; +import org.junit.jupiter.api.io.TempDir; + +/** + * Runs the {@link IcebergTableReadWriteTest} suite against external Iceberg tables: each table is + * created with an explicit {@code LOCATION} under a local temp directory, so Unity Catalog's + * Iceberg REST catalog registers it as EXTERNAL. Mirrors {@code DeltaExternalTableReadWriteTest}. + * + *

Only the local {@code file://} scheme is exercised (the shared base uses a file warehouse); + * the cloud schemes the UC-connector external tests cover would need an S3-protocol backend, which + * is out of scope here. + */ +public class IcebergExternalTableReadWriteTest extends IcebergTableReadWriteTest { + + @TempDir protected File dataDir; + + @Override + protected boolean isManagedTable() { + return false; + } + + @SneakyThrows + private String getLocation(TableSetupOptions options) { + String rawCatalogName = options.getCatalogName().replace("`", ""); + String rawTableName = options.getTableName().replace("`", ""); + return new File(new File(dataDir, rawCatalogName), rawTableName).getCanonicalPath(); + } + + @Override + protected String setupTable(TableSetupOptions options) { + sql(options.createExternalTableSql(getLocation(options))); + return options.fullTableName(); + } +} diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergManagedTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergManagedTableReadWriteTest.java new file mode 100644 index 0000000000..9fe4db0b8c --- /dev/null +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergManagedTableReadWriteTest.java @@ -0,0 +1,20 @@ +package io.unitycatalog.spark; + +/** + * Runs the {@link IcebergTableReadWriteTest} suite against managed Iceberg tables: the table has no + * client-supplied location, so Unity Catalog's Iceberg REST catalog assigns one under the managed + * storage root. Mirrors {@code DeltaManagedTableReadWriteTest}. + */ +public class IcebergManagedTableReadWriteTest extends IcebergTableReadWriteTest { + + @Override + protected boolean isManagedTable() { + return true; + } + + @Override + protected String setupTable(TableSetupOptions options) { + sql(options.createManagedTableSql()); + return options.fullTableName(); + } +} diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java new file mode 100644 index 0000000000..8d2332028e --- /dev/null +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java @@ -0,0 +1,203 @@ +package io.unitycatalog.spark; + +import static io.unitycatalog.server.utils.TestUtils.CATALOG_NAME; +import static org.assertj.core.api.Assertions.assertThat; + +import io.unitycatalog.client.ApiException; +import io.unitycatalog.client.model.ColumnInfo; +import io.unitycatalog.client.model.ColumnTypeName; +import io.unitycatalog.client.model.TableInfo; +import io.unitycatalog.server.utils.ServerProperties; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.spark.sql.Row; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.provider.Arguments; + +/** + * Shared base for the Spark + Iceberg end-to-end suites. It drives Iceberg tables through a real + * Spark + Iceberg runtime against Unity Catalog's Iceberg REST catalog ({@code + * /api/2.1/unity-catalog/iceberg}), rather than through the UC Spark connector (which has no + * Iceberg support). Spark is configured with Iceberg's own {@code SparkCatalog} (REST), wired by + * {@code BaseSparkIntegrationTest} for every catalog its {@code isIcebergCatalog} marks; the {@code + * /v1/config} handshake returns a {@code prefix} so the standard Iceberg client targets UC's + * catalog-scoped paths, exercising the server's config / namespace / create / commit / load / drop + * endpoints exactly as an external Iceberg client would. + * + *

Storage is a local {@code file://} warehouse: the server writes the first metadata file + * through its native local FileIO and Spark's Iceberg {@code HadoopFileIO} reads and writes data + * files and later metadata to the same directory, so the full create / write / read / commit path + * round-trips without cloud credentials. Cloud credential-vending is covered separately by the + * server-side Iceberg REST catalog tests. + * + *

Concrete subclasses pick managed ({@link IcebergManagedTableReadWriteTest}) or external + * ({@link IcebergExternalTableReadWriteTest}) tables, mirroring the Delta {@code + * DeltaManagedTableReadWriteTest} / {@code DeltaExternalTableReadWriteTest} split. The class reuses + * {@link BaseTableReadWriteTest}'s create/read/write matrix and helpers; the {@code + * ExternalTableReadWriteTest} base is UC-connector-specific, so it is not reused here. + * + *

Compiled only for Spark 4.0 / 4.1: Iceberg 1.11.0 publishes no Spark 4.2 runtime, so this + * source lives under {@code src/test/scala-shims/spark-4.0-4.1} and is absent from the 4.2 build. + */ +public abstract class IcebergTableReadWriteTest extends BaseTableReadWriteTest { + + @Override + protected boolean isIcebergCatalog(String catalog) { + return true; + } + + @Override + protected void setUpProperties() { + super.setUpProperties(); + // Advertise and accept the Iceberg write endpoints (createTable / updateTable / dropTable); + // otherwise the REST client refuses them as unsupported and the server rejects them. + serverProperties.setProperty(ServerProperties.Property.ICEBERG_TABLE_ENABLED.getKey(), "true"); + } + + @Override + protected String tableFormat() { + return "ICEBERG"; + } + + // These list only the named catalog; the Iceberg suites leave spark_catalog as Spark's built-in + // session catalog (the UC-connector tests also drive spark_catalog). + @Override + protected List supportedCatalogNames() { + return List.of(CATALOG_NAME); + } + + @Override + protected List sessionCatalogNames() { + return List.of(CATALOG_NAME); + } + + // Iceberg supports partitioned creates and CTAS, so every create in the base matrix is expected + // to succeed (the base defaults non-Delta CTAS / non-plain-CREATE to an expected failure). + @Override + protected List expectedCreateFailureMessages(TableSetupOptions options) { + return null; + } + + // The Iceberg REST catalog vends no cloud credentials and these suites use a local warehouse, so + // testTableOperations (and the other cloud-parameterized base tests) run once on the file scheme; + // the base guards its UC-connector-only steps behind !testingIceberg(). + protected static Stream cloudParameters() { + return Stream.of(Arguments.of("file", false, false)); + } + + /** One column spec for {@link #testTableWithSupportedDataTypes()}. */ + private static final class ColSpec { + private final String name; + private final String sqlType; + private final String insertValue; + private final String rowValue; + private final ColumnTypeName typeName; + + private ColSpec( + String name, + String sqlType, + String insertValue, + String rowValue, + ColumnTypeName typeName) { + this.name = name; + this.sqlType = sqlType; + this.insertValue = insertValue; + this.rowValue = rowValue; + this.typeName = typeName; + } + } + + /** + * Reuses the base data-type slot but with an Iceberg-appropriate type set: covers every Iceberg + * primitive reachable from Spark SQL (int, long, float, double, decimal, string, binary, boolean, + * date, timestamp, timestamp_ntz) plus the nested types (array, map, struct). + * Out of scope: Spark's TINYINT / SMALLINT (map to Iceberg int), CHAR / VARCHAR (to string), + * and TIME / UUID / FIXED / VARIANT (no Spark-SQL-reachable Iceberg equivalent). Asserts both + * the Spark-visible row values and the column names / type names UC persisted from the Iceberg + * schema. + */ + @Override + @Test + public void testTableWithSupportedDataTypes() throws ApiException { + List cols = + List.of( + new ColSpec("c_int", "INT", "1000", "1000", ColumnTypeName.INT), + new ColSpec("c_long", "BIGINT", "100000", "100000", ColumnTypeName.LONG), + new ColSpec("c_float", "FLOAT", "2.5", "2.5", ColumnTypeName.FLOAT), + new ColSpec("c_double", "DOUBLE", "1.5", "1.5", ColumnTypeName.DOUBLE), + new ColSpec("c_decimal", "DECIMAL(10,2)", "123.45", "123.45", ColumnTypeName.DECIMAL), + new ColSpec("c_string", "STRING", "'test'", "test", ColumnTypeName.STRING), + // BINARY round-trips as a Java byte[]; its toString() is an object ref, so rowValue is + // unused and the value check below matches the "[B@" prefix instead. + new ColSpec("c_binary", "BINARY", "X'CAFEBABE'", null, ColumnTypeName.BINARY), + new ColSpec("c_boolean", "BOOLEAN", "true", "true", ColumnTypeName.BOOLEAN), + new ColSpec("c_date", "DATE", "DATE'2025-01-01'", "2025-01-01", ColumnTypeName.DATE), + new ColSpec( + "c_timestamp", + "TIMESTAMP", + "TIMESTAMP'2025-01-01 12:00:00'", + "2025-01-01 12:00:00.0", + ColumnTypeName.TIMESTAMP), + new ColSpec( + "c_timestamp_ntz", + "TIMESTAMP_NTZ", + "TIMESTAMP_NTZ'2025-01-01 12:00:00'", + "2025-01-01T12:00", + ColumnTypeName.TIMESTAMP_NTZ), + new ColSpec( + "c_arr", "ARRAY", "array(1, 2, 3)", "ArraySeq(1, 2, 3)", ColumnTypeName.ARRAY), + new ColSpec( + "c_map", "MAP", "map('k', 10)", "Map(k -> 10)", ColumnTypeName.MAP), + new ColSpec( + "c_struct", + "STRUCT", + "struct(42, 'x')", + "[42,x]", + ColumnTypeName.STRUCT)); + + session = createSparkSessionWithCatalogs(CATALOG_NAME); + String tableName = TEST_TABLE + "_types"; + String fullTableName = + setupTable( + new TableSetupOptions() + .setCatalogName(CATALOG_NAME) + .setTableName(tableName) + .setColumns( + cols.stream() + .map(c -> Pair.of(c.name, c.sqlType)) + .collect(Collectors.toList()))); + + String colNames = cols.stream().map(c -> c.name).collect(Collectors.joining(", ")); + sql( + "INSERT INTO %s (%s) VALUES (%s)", + fullTableName, + colNames, + cols.stream().map(c -> c.insertValue).collect(Collectors.joining(", "))); + + List queryResult = sql("SELECT %s FROM %s", colNames, fullTableName); + assertThat(queryResult).hasSize(1); + Row row = queryResult.get(0); + for (int i = 0; i < cols.size(); i++) { + ColSpec spec = cols.get(i); + if (spec.typeName == ColumnTypeName.BINARY) { + assertThat(row.get(i).toString()).as("row value for %s", spec.name).startsWith("[B@"); + } else { + assertThat(row.get(i).toString()) + .as("row value for %s", spec.name) + .isEqualTo(spec.rowValue); + } + } + + TableInfo tableInfo = tableOperations.getTable(fullTableName); + List columns = tableInfo.getColumns(); + assertThat(columns).hasSize(cols.size()); + for (int i = 0; i < cols.size(); i++) { + assertThat(columns.get(i).getName()).as("name[%d]", i).isEqualTo(cols.get(i).name); + assertThat(columns.get(i).getTypeName()) + .as("typeName for %s", cols.get(i).name) + .isEqualTo(cols.get(i).typeName); + } + } +} From 7499d29fb34c479eb234e225d251f68704f1a4e5 Mon Sep 17 00:00:00 2001 From: Yi Li Date: Thu, 1 Oct 2026 10:42:31 -0700 Subject: [PATCH 2/6] Address comments Signed-off-by: Yi Li --- .../spark/BaseTableReadWriteTest.java | 53 +++++++++++-------- .../spark/IcebergTableReadWriteTest.java | 10 ++-- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java index 7fd80739a3..989d6fe512 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java @@ -29,6 +29,7 @@ import org.apache.spark.sql.Row; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.StructType; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIf; @@ -478,14 +479,8 @@ public void testTableOperations( .map(row -> row.getString(1)) .collect(Collectors.toList()); assertThat(tableNames).containsExactlyInAnyOrder(TEST_TABLE, ANOTHER_TEST_TABLE); - // A nested (multi-level) namespace is rejected by the Iceberg REST catalog. Asserted by class - // name, not an imported type, so this shared base still compiles on Spark 4.2 (no Iceberg on - // its classpath). - assertThatThrownBy(() -> sql("SHOW TABLES in %s.a.b", CATALOG_NAME)) - .satisfies( - t -> - assertThat(t.getClass().getName()) - .isEqualTo("org.apache.iceberg.exceptions.RESTException")); + // A multi-level namespace does not exist as a UC schema, so listing under it is rejected. + assertIcebergSchemaNotFound(() -> sql("SHOW TABLES in %s.a.b", CATALOG_NAME)); } else { List tables1 = sql("SHOW TABLES in %s.%s", SPARK_CATALOG, SCHEMA_NAME); assertThat(tables1).hasSize(1); @@ -509,11 +504,8 @@ public void testTableOperations( } // Dropping a malformed multi-level table identifier is rejected. if (testingIceberg()) { - assertThatThrownBy(() -> sql("DROP TABLE %s.a.b.c", CATALOG_NAME)) - .satisfies( - t -> - assertThat(t.getClass().getName()) - .isEqualTo("org.apache.iceberg.exceptions.RESTException")); + // The namespace a.b does not exist as a UC schema, so the drop is rejected before the table. + assertIcebergSchemaNotFound(() -> sql("DROP TABLE %s.a.b.c", CATALOG_NAME)); } else { // Delta < 4.3.0 forwards the 4-part identifier to UC, which rejects it server-side with // ApiException("Nested namespaces are not supported"). Delta >= 4.3.0 ships @@ -594,15 +586,15 @@ public void testHyphenInTableName() { /** * Specification for one column in testTableWithSupportedDataTypes: its SQL DDL type, the SQL - * literal used in the INSERT, the expected toString() from the result row (null = byte-array ref - * check via startsWith("[B@")), and the expected UC catalog metadata fields. + * literal used in the INSERT, the expected toString() from the result row (null for BINARY, whose + * raw bytes are compared directly), and the expected UC catalog metadata fields. */ @Getter private static final class ColSpec { private final String name; private final String sqlType; private final String insertValue; - private final String rowValue; // null = byte-array object ref, checked with startsWith + private final String rowValue; // null for BINARY, whose bytes are compared directly private final ColumnTypeName typeName; private final String typeText; private final boolean nullable; @@ -882,10 +874,10 @@ public void testTableWithSupportedDataTypes() throws ApiException { List queryResult = sql("SELECT %s FROM %s", colNames, fullTableName); assertThat(queryResult).hasSize(1); + Row resultRow = queryResult.get(0); List row = - IntStream.range(0, queryResult.get(0).length()) - .mapToObj( - i -> queryResult.get(0).isNullAt(i) ? null : queryResult.get(0).get(i).toString()) + IntStream.range(0, resultRow.length()) + .mapToObj(i -> resultRow.isNullAt(i) ? null : resultRow.get(i).toString()) .collect(Collectors.toList()); TableInfo tableInfo = tableOperations.getTable(fullTableName); @@ -910,8 +902,11 @@ public void testTableWithSupportedDataTypes() throws ApiException { for (int i = 0; i < cols.size(); i++) { ColSpec spec = cols.get(i); if (spec.getTypeName() == ColumnTypeName.BINARY) { - // BINARY: row value is a Java byte array — toString() produces a ref like "[B@..." - assertThat(row.get(i)).as("row value for %s", spec.getName()).startsWith("[B@"); + // BINARY comes back as a Java byte[]; read it from the raw row (the stringified `row` above + // only keeps its "[B@" object ref) and check the inserted X'CAFEBABE' round-trips exactly. + assertThat((byte[]) resultRow.get(i)) + .as("row value for %s", spec.getName()) + .containsExactly(0xCA, 0xFE, 0xBA, 0xBE); } else { assertThat(row.get(i)).as("row value for %s", spec.getName()).isEqualTo(spec.getRowValue()); } @@ -981,4 +976,20 @@ protected void testTableReadWriteCreatedAsSelect( sql("INSERT INTO %s SELECT %d, '%s'", tableFullName, newValue.getLeft(), newValue.getRight()); validateRows(sql("SELECT * FROM %s ORDER BY i", tableFullName), asSelect, newValue); } + + /** + * Asserts {@code op} is rejected specifically because its multi-level namespace does not exist as + * a UC schema. The Iceberg REST client maps UC's rejection to {@code NoSuchNamespaceException}; + * checked by class name (not an imported type, so this shared base still compiles on Spark 4.2, + * which has no Iceberg on its classpath) and by message, so an unrelated non-2xx (a 500 or auth + * failure, which also surfaces as a {@code RESTException}) does not make the assertion pass. + */ + private static void assertIcebergSchemaNotFound(ThrowingCallable op) { + assertThatThrownBy(op) + .hasMessageContaining("Schema not found") + .satisfies( + t -> + assertThat(t.getClass().getName()) + .isEqualTo("org.apache.iceberg.exceptions.NoSuchNamespaceException")); + } } diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java index 8d2332028e..112ac54665 100644 --- a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java @@ -129,8 +129,8 @@ public void testTableWithSupportedDataTypes() throws ApiException { new ColSpec("c_double", "DOUBLE", "1.5", "1.5", ColumnTypeName.DOUBLE), new ColSpec("c_decimal", "DECIMAL(10,2)", "123.45", "123.45", ColumnTypeName.DECIMAL), new ColSpec("c_string", "STRING", "'test'", "test", ColumnTypeName.STRING), - // BINARY round-trips as a Java byte[]; its toString() is an object ref, so rowValue is - // unused and the value check below matches the "[B@" prefix instead. + // BINARY round-trips as a Java byte[], not a String, so rowValue is unused; the check + // below compares the raw bytes against the inserted X'CAFEBABE' instead. new ColSpec("c_binary", "BINARY", "X'CAFEBABE'", null, ColumnTypeName.BINARY), new ColSpec("c_boolean", "BOOLEAN", "true", "true", ColumnTypeName.BOOLEAN), new ColSpec("c_date", "DATE", "DATE'2025-01-01'", "2025-01-01", ColumnTypeName.DATE), @@ -182,7 +182,11 @@ public void testTableWithSupportedDataTypes() throws ApiException { for (int i = 0; i < cols.size(); i++) { ColSpec spec = cols.get(i); if (spec.typeName == ColumnTypeName.BINARY) { - assertThat(row.get(i).toString()).as("row value for %s", spec.name).startsWith("[B@"); + // Verify the inserted X'CAFEBABE' survived the round-trip byte-for-byte; toString() on a + // byte[] is only an object ref and would assert nothing about the value. + assertThat((byte[]) row.get(i)) + .as("row value for %s", spec.name) + .containsExactly(0xCA, 0xFE, 0xBA, 0xBE); } else { assertThat(row.get(i).toString()) .as("row value for %s", spec.name) From cf4f4b5ce199b124caf38758cbe7aa525c8c3c61 Mon Sep 17 00:00:00 2001 From: Yi Li Date: Thu, 1 Oct 2026 20:57:21 -0700 Subject: [PATCH 3/6] Address comments Signed-off-by: Yi Li --- .../spark/BaseSparkIntegrationTest.java | 2 +- .../spark/BaseTableReadWriteTest.java | 18 +++++++++++++----- .../spark/UCTablePropertiesTest.java | 3 ++- .../spark/IcebergTableReadWriteTest.java | 6 +----- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java index a1e9574754..c8191e1200 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java @@ -67,7 +67,7 @@ protected SparkSession createSparkSessionWithCatalogs(String... catalogs) { } /** The base Spark builder shared by all integration tests (local master, small shuffle width). */ - protected SparkSession.Builder newSparkSessionBuilder() { + private SparkSession.Builder newSparkSessionBuilder() { return SparkSession.builder() .appName("test") .master("local[*]") diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java index 989d6fe512..3a256933c7 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java @@ -902,11 +902,8 @@ public void testTableWithSupportedDataTypes() throws ApiException { for (int i = 0; i < cols.size(); i++) { ColSpec spec = cols.get(i); if (spec.getTypeName() == ColumnTypeName.BINARY) { - // BINARY comes back as a Java byte[]; read it from the raw row (the stringified `row` above - // only keeps its "[B@" object ref) and check the inserted X'CAFEBABE' round-trips exactly. - assertThat((byte[]) resultRow.get(i)) - .as("row value for %s", spec.getName()) - .containsExactly(0xCA, 0xFE, 0xBA, 0xBE); + // Read BINARY from the raw row; stringified `row` only keeps its "[B@" ref. + assertCafebabeBinary(resultRow.get(i), spec.getName()); } else { assertThat(row.get(i)).as("row value for %s", spec.getName()).isEqualTo(spec.getRowValue()); } @@ -992,4 +989,15 @@ private static void assertIcebergSchemaNotFound(ThrowingCallable op) { assertThat(t.getClass().getName()) .isEqualTo("org.apache.iceberg.exceptions.NoSuchNamespaceException")); } + + /** + * Asserts a BINARY column round-tripped the inserted {@code X'CAFEBABE'} byte-for-byte. Shared + * with the {@code testTableWithSupportedDataTypes} override so the expected bytes live in one + * place; {@code rowValue} is the raw row cell, which is a Java {@code byte[]} for BINARY. + */ + protected static void assertCafebabeBinary(Object rowValue, String columnName) { + assertThat((byte[]) rowValue) + .as("row value for %s", columnName) + .containsExactly(0xCA, 0xFE, 0xBA, 0xBE); + } } diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/UCTablePropertiesTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/UCTablePropertiesTest.java index 2608aec1b4..7d01920405 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/UCTablePropertiesTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/UCTablePropertiesTest.java @@ -52,7 +52,8 @@ public void shouldPersistProperty_dropsReservedCredentialsAndSchemaKeys() { UCTableProperties.shouldPersistProperty( TableCatalog.OPTION_PREFIX + "fs.s3a.session.token")) .isFalse(); - assertThat(UCTableProperties.shouldPersistProperty("spark.sql.sources.schema.part.0")) + assertThat( + UCTableProperties.shouldPersistProperty("spark.sql.sources.schema.part.0")) .isFalse(); assertThat( UCTableProperties.shouldPersistProperty( diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java index 112ac54665..9a87196936 100644 --- a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java @@ -182,11 +182,7 @@ public void testTableWithSupportedDataTypes() throws ApiException { for (int i = 0; i < cols.size(); i++) { ColSpec spec = cols.get(i); if (spec.typeName == ColumnTypeName.BINARY) { - // Verify the inserted X'CAFEBABE' survived the round-trip byte-for-byte; toString() on a - // byte[] is only an object ref and would assert nothing about the value. - assertThat((byte[]) row.get(i)) - .as("row value for %s", spec.name) - .containsExactly(0xCA, 0xFE, 0xBA, 0xBE); + assertCafebabeBinary(row.get(i), spec.name); } else { assertThat(row.get(i).toString()) .as("row value for %s", spec.name) From 4ddea1ae740efcbf6474f6915fbff564d0b1a278 Mon Sep 17 00:00:00 2001 From: Yi Li Date: Thu, 1 Oct 2026 22:46:06 -0700 Subject: [PATCH 4/6] Enable more tests Signed-off-by: Yi Li --- .../java/io/unitycatalog/spark/BaseTableReadWriteTest.java | 5 +++-- .../io/unitycatalog/spark/IcebergTableReadWriteTest.java | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java index 3a256933c7..f619e45e51 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java @@ -455,8 +455,9 @@ public void testTableOperations( sql("INSERT INTO %s SELECT 2, 'a'", t2); validateRows(sql("SELECT * FROM %s", t2), Pair.of(2, "a")); - if (testingDelta()) { - // UPDATE, MERGE INTO and DELETE are only supported in Delta tables. + if (testingDelta() || testingIceberg()) { + // UPDATE, MERGE INTO and DELETE need a format with row-level support (Delta or Iceberg); the + // external-Parquet path does not support them. // Test UPDATE. The table t2 will have (2, 'b') sql("UPDATE %s SET s = 'b' WHERE i = 2", t2); diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java index 9a87196936..68566c59ac 100644 --- a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java @@ -6,7 +6,9 @@ import io.unitycatalog.client.ApiException; import io.unitycatalog.client.model.ColumnInfo; import io.unitycatalog.client.model.ColumnTypeName; +import io.unitycatalog.client.model.DataSourceFormat; import io.unitycatalog.client.model.TableInfo; +import io.unitycatalog.client.model.TableType; import io.unitycatalog.server.utils.ServerProperties; import java.util.List; import java.util.stream.Collectors; @@ -191,6 +193,11 @@ public void testTableWithSupportedDataTypes() throws ApiException { } TableInfo tableInfo = tableOperations.getTable(fullTableName); + // A client-supplied LOCATION registers the table EXTERNAL, otherwise MANAGED; the format is + // always ICEBERG. Confirms the managed/external subclasses produce what they claim. + assertThat(tableInfo.getTableType()) + .isEqualTo(isManagedTable() ? TableType.MANAGED : TableType.EXTERNAL); + assertThat(tableInfo.getDataSourceFormat()).isEqualTo(DataSourceFormat.ICEBERG); List columns = tableInfo.getColumns(); assertThat(columns).hasSize(cols.size()); for (int i = 0; i < cols.size(); i++) { From 2d08b120bf96c183e9baee86930498eab0f1b21e Mon Sep 17 00:00:00 2001 From: Yi Li Date: Fri, 2 Oct 2026 11:38:45 -0700 Subject: [PATCH 5/6] Also test typeJson etc for Iceberg Signed-off-by: Yi Li --- .../spark/BaseTableReadWriteTest.java | 2 +- .../spark/IcebergTableReadWriteTest.java | 108 +++++++++++++++--- 2 files changed, 90 insertions(+), 20 deletions(-) diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java index f619e45e51..18ebcbb481 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java @@ -633,7 +633,7 @@ private ColSpec( } } - private static String structFieldTypeJson(String name, String dataTypeJson) { + protected static String structFieldTypeJson(String name, String dataTypeJson) { return structFieldTypeJson(name, dataTypeJson, true, "{}"); } diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java index 68566c59ac..589d36e679 100644 --- a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java @@ -96,18 +96,26 @@ private static final class ColSpec { private final String insertValue; private final String rowValue; private final ColumnTypeName typeName; + // How UC persists the type: typeText is the SQL type string, dataTypeJson the inner Spark + // DataType JSON (wrapped into the struct-field JSON for the typeJson assertion). + private final String typeText; + private final String dataTypeJson; private ColSpec( String name, String sqlType, String insertValue, String rowValue, - ColumnTypeName typeName) { + ColumnTypeName typeName, + String typeText, + String dataTypeJson) { this.name = name; this.sqlType = sqlType; this.insertValue = insertValue; this.rowValue = rowValue; this.typeName = typeName; + this.typeText = typeText; + this.dataTypeJson = dataTypeJson; } } @@ -123,41 +131,99 @@ private ColSpec( @Override @Test public void testTableWithSupportedDataTypes() throws ApiException { + String arrJson = "{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true}"; + String mapJson = + "{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"integer\"," + + "\"valueContainsNull\":true}"; + String structJson = + "{\"type\":\"struct\",\"fields\":[" + + "{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}," + + "{\"name\":\"b\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"; List cols = List.of( - new ColSpec("c_int", "INT", "1000", "1000", ColumnTypeName.INT), - new ColSpec("c_long", "BIGINT", "100000", "100000", ColumnTypeName.LONG), - new ColSpec("c_float", "FLOAT", "2.5", "2.5", ColumnTypeName.FLOAT), - new ColSpec("c_double", "DOUBLE", "1.5", "1.5", ColumnTypeName.DOUBLE), - new ColSpec("c_decimal", "DECIMAL(10,2)", "123.45", "123.45", ColumnTypeName.DECIMAL), - new ColSpec("c_string", "STRING", "'test'", "test", ColumnTypeName.STRING), + new ColSpec("c_int", "INT", "1000", "1000", ColumnTypeName.INT, "int", "\"integer\""), + new ColSpec( + "c_long", "BIGINT", "100000", "100000", ColumnTypeName.LONG, "bigint", "\"long\""), + new ColSpec( + "c_float", "FLOAT", "2.5", "2.5", ColumnTypeName.FLOAT, "float", "\"float\""), + new ColSpec( + "c_double", "DOUBLE", "1.5", "1.5", ColumnTypeName.DOUBLE, "double", "\"double\""), + new ColSpec( + "c_decimal", + "DECIMAL(10,2)", + "123.45", + "123.45", + ColumnTypeName.DECIMAL, + "decimal(10,2)", + "\"decimal(10,2)\""), + new ColSpec( + "c_string", "STRING", "'test'", "test", ColumnTypeName.STRING, "string", "\"string\""), // BINARY round-trips as a Java byte[], not a String, so rowValue is unused; the check // below compares the raw bytes against the inserted X'CAFEBABE' instead. - new ColSpec("c_binary", "BINARY", "X'CAFEBABE'", null, ColumnTypeName.BINARY), - new ColSpec("c_boolean", "BOOLEAN", "true", "true", ColumnTypeName.BOOLEAN), - new ColSpec("c_date", "DATE", "DATE'2025-01-01'", "2025-01-01", ColumnTypeName.DATE), + new ColSpec( + "c_binary", + "BINARY", + "X'CAFEBABE'", + null, + ColumnTypeName.BINARY, + "binary", + "\"binary\""), + new ColSpec( + "c_boolean", + "BOOLEAN", + "true", + "true", + ColumnTypeName.BOOLEAN, + "boolean", + "\"boolean\""), + new ColSpec( + "c_date", + "DATE", + "DATE'2025-01-01'", + "2025-01-01", + ColumnTypeName.DATE, + "date", + "\"date\""), new ColSpec( "c_timestamp", "TIMESTAMP", "TIMESTAMP'2025-01-01 12:00:00'", "2025-01-01 12:00:00.0", - ColumnTypeName.TIMESTAMP), + ColumnTypeName.TIMESTAMP, + "timestamp", + "\"timestamp\""), new ColSpec( "c_timestamp_ntz", "TIMESTAMP_NTZ", "TIMESTAMP_NTZ'2025-01-01 12:00:00'", "2025-01-01T12:00", - ColumnTypeName.TIMESTAMP_NTZ), + ColumnTypeName.TIMESTAMP_NTZ, + "timestamp_ntz", + "\"timestamp_ntz\""), new ColSpec( - "c_arr", "ARRAY", "array(1, 2, 3)", "ArraySeq(1, 2, 3)", ColumnTypeName.ARRAY), + "c_arr", + "ARRAY", + "array(1, 2, 3)", + "ArraySeq(1, 2, 3)", + ColumnTypeName.ARRAY, + "array", + arrJson), new ColSpec( - "c_map", "MAP", "map('k', 10)", "Map(k -> 10)", ColumnTypeName.MAP), + "c_map", + "MAP", + "map('k', 10)", + "Map(k -> 10)", + ColumnTypeName.MAP, + "map", + mapJson), new ColSpec( "c_struct", "STRUCT", "struct(42, 'x')", "[42,x]", - ColumnTypeName.STRUCT)); + ColumnTypeName.STRUCT, + "struct", + structJson)); session = createSparkSessionWithCatalogs(CATALOG_NAME); String tableName = TEST_TABLE + "_types"; @@ -201,10 +267,14 @@ public void testTableWithSupportedDataTypes() throws ApiException { List columns = tableInfo.getColumns(); assertThat(columns).hasSize(cols.size()); for (int i = 0; i < cols.size(); i++) { - assertThat(columns.get(i).getName()).as("name[%d]", i).isEqualTo(cols.get(i).name); - assertThat(columns.get(i).getTypeName()) - .as("typeName for %s", cols.get(i).name) - .isEqualTo(cols.get(i).typeName); + ColumnInfo col = columns.get(i); + ColSpec spec = cols.get(i); + assertThat(col.getName()).as("name[%d]", i).isEqualTo(spec.name); + assertThat(col.getTypeName()).as("typeName for %s", spec.name).isEqualTo(spec.typeName); + assertThat(col.getTypeText()).as("typeText for %s", spec.name).isEqualTo(spec.typeText); + assertThat(col.getTypeJson()) + .as("typeJson for %s", spec.name) + .isEqualTo(structFieldTypeJson(spec.name, spec.dataTypeJson)); } } } From ca7f8b9c7f1d4c1592f7e9a5ad9a93aa37f0ee50 Mon Sep 17 00:00:00 2001 From: Yi Li Date: Fri, 2 Oct 2026 14:06:13 -0700 Subject: [PATCH 6/6] Address comments Signed-off-by: Yi Li --- build.sbt | 23 +- .../spark/BaseTableReadWriteTest.java | 52 +++-- .../DeltaIcebergCrossFormatJoinTest.java | 5 +- .../spark/IcebergTableReadWriteTest.java | 207 +----------------- project/CrossSparkVersions.scala | 47 +++- project/spark-versions.json | 6 +- 6 files changed, 87 insertions(+), 253 deletions(-) diff --git a/build.sbt b/build.sbt index 7d728085c9..9728f63322 100644 --- a/build.sbt +++ b/build.sbt @@ -26,24 +26,6 @@ lazy val deltaVersion = sys.props.getOrElse("deltaVersion", "4.3.1") // This String val is used for libraryDependencies coordinates; the SettingKey is // queryable in SBT via `show spark/sparkVersion`. lazy val sparkVersion = CrossSparkVersions.getSparkArtifactVersion() -lazy val sparkMajorMinorVersion = CrossSparkVersions.getSparkVersionSpec().shortVersion - -// delta-spark is only needed for tests. When UC is published to local Maven before -// Delta is built (e.g. CI pre-Delta publishM2 step), the matching Delta artifact may -// not exist yet. Pass -DskipDeltaSpark=true to exclude it and avoid resolution failures. -def deltaSparkTestDeps: Seq[ModuleID] = - if (sys.props.getOrElse("skipDeltaSpark", "false").toBoolean) Seq.empty - else Seq("io.delta" %% s"delta-spark_$sparkMajorMinorVersion" % deltaVersion % Test) - -// iceberg-spark-runtime is published for Spark 4.0 and 4.1 (Iceberg 1.11.0) but not Spark 4.2, so -// it is added only for the versions that have a runtime. The Iceberg-on-Spark integration test -// source lives under src/test/scala-shims/spark-4.0-4.1, which is compiled only for those -// versions, so nothing references this dep on Spark 4.2. -def icebergSparkTestDeps: Seq[ModuleID] = - if (CrossSparkVersions.getSparkVersionSpec().isAtLeast(4, 2)) Seq.empty - else - Seq( - "org.apache.iceberg" % s"iceberg-spark-runtime-${sparkMajorMinorVersion}_2.13" % icebergVersion % Test) // Apache Snapshots resolver is in build/sbt-config/repositories (global). // No per-module sparkResolvers needed. @@ -755,7 +737,8 @@ lazy val spark = (project in file("connectors/spark")) "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % Test, "org.projectlombok" % "lombok" % "1.18.32" % Test, "com.google.cloud.bigdataoss" % "gcs-connector" % "3.0.2" % Test classifier "shaded", - ) ++ deltaSparkTestDeps ++ icebergSparkTestDeps, + ) ++ CrossSparkVersions.deltaSparkTestDeps(deltaVersion) + ++ CrossSparkVersions.icebergSparkTestDeps(icebergVersion), dependencyOverrides ++= Seq( "com.fasterxml.jackson.core" % "jackson-databind" % sparkJacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkJacksonVersion, @@ -845,7 +828,7 @@ lazy val integrationTests = (project in file("integration-tests")) "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % Test, "org.apache.hadoop" % "hadoop-azure" % hadoopVersion % Test, "com.google.cloud.bigdataoss" % "gcs-connector" % "3.0.2" % Test classifier "shaded", - ) ++ deltaSparkTestDeps, + ) ++ CrossSparkVersions.deltaSparkTestDeps(deltaVersion), dependencyOverrides ++= Seq( "com.fasterxml.jackson.core" % "jackson-databind" % sparkJacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkJacksonVersion, diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java index 18ebcbb481..3328168768 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java @@ -10,7 +10,9 @@ import io.unitycatalog.client.ApiException; import io.unitycatalog.client.model.ColumnInfo; import io.unitycatalog.client.model.ColumnTypeName; +import io.unitycatalog.client.model.DataSourceFormat; import io.unitycatalog.client.model.TableInfo; +import io.unitycatalog.client.model.TableType; import io.unitycatalog.server.base.table.TableOperations; import io.unitycatalog.server.persist.utils.PagedListingHelper; import io.unitycatalog.server.sdk.tables.SdkTableOperations; @@ -439,15 +441,15 @@ public void testTimeTravelDeltaTable() { @MethodSource("cloudParameters") public void testTableOperations( String scheme, boolean renewCredEnabled, boolean credScopedFsEnabled) { - // The UC connector drives both spark_catalog and a named catalog; Iceberg wires only the named - // catalog (spark_catalog stays Spark's built-in session catalog), so it puts both tables there. - String[] catalogs = - testingIceberg() ? new String[] {CATALOG_NAME} : new String[] {SPARK_CATALOG, CATALOG_NAME}; - String firstCatalog = testingIceberg() ? CATALOG_NAME : SPARK_CATALOG; - session = createSparkSessionWithCatalogs(renewCredEnabled, credScopedFsEnabled, catalogs); + // t1 goes in the first session catalog: spark_catalog for the UC connector, or the named + // catalog when that is the only one wired (Iceberg), so both tables then share it. + List catalogs = sessionCatalogNames(); + session = + createSparkSessionWithCatalogs( + renewCredEnabled, credScopedFsEnabled, catalogs.toArray(new String[0])); // t1 has (1, 'a') - String t1 = setupTable(scheme, firstCatalog, TEST_TABLE); + String t1 = setupTable(scheme, catalogs.get(0), TEST_TABLE); testTableReadWrite(t1); // t2 has (2, 'a') @@ -530,7 +532,7 @@ public void testTableOperations( // Asserting all 3 names proves pagination worked end-to-end. @Test public void testListTablesPagination() { - String catalog = testingIceberg() ? CATALOG_NAME : SPARK_CATALOG; + String catalog = sessionCatalogNames().get(0); session = createSparkSessionWithCatalogs(catalog); Integer originalPageSize = PagedListingHelper.DEFAULT_PAGE_SIZE; try { @@ -633,10 +635,6 @@ private ColSpec( } } - protected static String structFieldTypeJson(String name, String dataTypeJson) { - return structFieldTypeJson(name, dataTypeJson, true, "{}"); - } - private static String structFieldTypeJson( String name, String dataTypeJson, boolean nullable, String metadataJson) { return String.format( @@ -851,10 +849,23 @@ public void testTableWithSupportedDataTypes() throws ApiException { .filter(col -> !col.getName().equals("col_variant")) .collect(Collectors.toList()); } + if (testingIceberg()) { + // Iceberg has no distinct TINYINT / SMALLINT (widened to int) or CHAR / VARCHAR (stored as + // string), and VARIANT needs format v3, so these would not round-trip as declared. + List unsupported = + List.of("col_tinyint", "col_smallint", "col_char", "col_varchar", "col_variant"); + cols = + cols.stream() + .filter(col -> !unsupported.contains(col.getName())) + .collect(Collectors.toList()); + } - session = createSparkSessionWithCatalogs(SPARK_CATALOG, CATALOG_NAME); + session = createSparkSessionWithCatalogs(sessionCatalogNames().toArray(new String[0])); String tableName = TEST_TABLE + "_complex_type"; - List partitionColumns = List.of("col_string", "col_bigint"); + // UC records no partitionIndex for Iceberg (its partition spec is not per-column), so Iceberg + // tables are created unpartitioned. + List partitionColumns = + testingIceberg() ? List.of() : List.of("col_string", "col_bigint"); String fullTableName = setupTable( new TableSetupOptions() @@ -882,6 +893,12 @@ public void testTableWithSupportedDataTypes() throws ApiException { .collect(Collectors.toList()); TableInfo tableInfo = tableOperations.getTable(fullTableName); + // A client-supplied LOCATION registers the table EXTERNAL, otherwise MANAGED; the format is the + // suite's tableFormat(). Confirms each managed/external subclass produces what it claims. + assertThat(tableInfo.getTableType()) + .isEqualTo(isManagedTable() ? TableType.MANAGED : TableType.EXTERNAL); + assertThat(tableInfo.getDataSourceFormat()) + .isEqualTo(DataSourceFormat.valueOf(tableFormat().toUpperCase())); List columns = tableInfo.getColumns(); assertThat(columns).hasSize(cols.size()); @@ -992,11 +1009,10 @@ private static void assertIcebergSchemaNotFound(ThrowingCallable op) { } /** - * Asserts a BINARY column round-tripped the inserted {@code X'CAFEBABE'} byte-for-byte. Shared - * with the {@code testTableWithSupportedDataTypes} override so the expected bytes live in one - * place; {@code rowValue} is the raw row cell, which is a Java {@code byte[]} for BINARY. + * Asserts a BINARY column round-tripped the inserted {@code X'CAFEBABE'} byte-for-byte; {@code + * rowValue} is the raw row cell, which is a Java {@code byte[]} for BINARY. */ - protected static void assertCafebabeBinary(Object rowValue, String columnName) { + private static void assertCafebabeBinary(Object rowValue, String columnName) { assertThat((byte[]) rowValue) .as("row value for %s", columnName) .containsExactly(0xCA, 0xFE, 0xBA, 0xBE); diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java index 934d38cae1..2a17cbf776 100644 --- a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java @@ -27,9 +27,8 @@ * shared {@code createSparkSessionWithCatalogs} derives both session extensions from the catalog * mix, and storage is a local {@code file://} warehouse. * - *

Compiled only for Spark 4.0 / 4.1: Iceberg 1.11.0 publishes no Spark 4.2 runtime, so this - * source lives under {@code src/test/scala-shims/spark-4.0-4.1} (see {@link - * IcebergTableReadWriteTest}). + *

Lives under {@code src/test/scala-shims/spark-4.0-4.1} so it is compiled only for the Spark + * versions that have an Iceberg Spark runtime ({@code supportIceberg} in the cross-Spark build). */ public class DeltaIcebergCrossFormatJoinTest extends BaseSparkIntegrationTest { diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java index 589d36e679..7970cbc6c0 100644 --- a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java @@ -1,21 +1,10 @@ package io.unitycatalog.spark; import static io.unitycatalog.server.utils.TestUtils.CATALOG_NAME; -import static org.assertj.core.api.Assertions.assertThat; -import io.unitycatalog.client.ApiException; -import io.unitycatalog.client.model.ColumnInfo; -import io.unitycatalog.client.model.ColumnTypeName; -import io.unitycatalog.client.model.DataSourceFormat; -import io.unitycatalog.client.model.TableInfo; -import io.unitycatalog.client.model.TableType; import io.unitycatalog.server.utils.ServerProperties; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.spark.sql.Row; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.provider.Arguments; /** @@ -40,8 +29,8 @@ * {@link BaseTableReadWriteTest}'s create/read/write matrix and helpers; the {@code * ExternalTableReadWriteTest} base is UC-connector-specific, so it is not reused here. * - *

Compiled only for Spark 4.0 / 4.1: Iceberg 1.11.0 publishes no Spark 4.2 runtime, so this - * source lives under {@code src/test/scala-shims/spark-4.0-4.1} and is absent from the 4.2 build. + *

Lives under {@code src/test/scala-shims/spark-4.0-4.1} so it is compiled only for the Spark + * versions that have an Iceberg Spark runtime ({@code supportIceberg} in the cross-Spark build). */ public abstract class IcebergTableReadWriteTest extends BaseTableReadWriteTest { @@ -83,198 +72,8 @@ protected List expectedCreateFailureMessages(TableSetupOptions options) } // The Iceberg REST catalog vends no cloud credentials and these suites use a local warehouse, so - // testTableOperations (and the other cloud-parameterized base tests) run once on the file scheme; - // the base guards its UC-connector-only steps behind !testingIceberg(). + // testTableOperations (and the other cloud-parameterized base tests) run once on the file scheme. protected static Stream cloudParameters() { return Stream.of(Arguments.of("file", false, false)); } - - /** One column spec for {@link #testTableWithSupportedDataTypes()}. */ - private static final class ColSpec { - private final String name; - private final String sqlType; - private final String insertValue; - private final String rowValue; - private final ColumnTypeName typeName; - // How UC persists the type: typeText is the SQL type string, dataTypeJson the inner Spark - // DataType JSON (wrapped into the struct-field JSON for the typeJson assertion). - private final String typeText; - private final String dataTypeJson; - - private ColSpec( - String name, - String sqlType, - String insertValue, - String rowValue, - ColumnTypeName typeName, - String typeText, - String dataTypeJson) { - this.name = name; - this.sqlType = sqlType; - this.insertValue = insertValue; - this.rowValue = rowValue; - this.typeName = typeName; - this.typeText = typeText; - this.dataTypeJson = dataTypeJson; - } - } - - /** - * Reuses the base data-type slot but with an Iceberg-appropriate type set: covers every Iceberg - * primitive reachable from Spark SQL (int, long, float, double, decimal, string, binary, boolean, - * date, timestamp, timestamp_ntz) plus the nested types (array, map, struct). - * Out of scope: Spark's TINYINT / SMALLINT (map to Iceberg int), CHAR / VARCHAR (to string), - * and TIME / UUID / FIXED / VARIANT (no Spark-SQL-reachable Iceberg equivalent). Asserts both - * the Spark-visible row values and the column names / type names UC persisted from the Iceberg - * schema. - */ - @Override - @Test - public void testTableWithSupportedDataTypes() throws ApiException { - String arrJson = "{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true}"; - String mapJson = - "{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"integer\"," - + "\"valueContainsNull\":true}"; - String structJson = - "{\"type\":\"struct\",\"fields\":[" - + "{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}," - + "{\"name\":\"b\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"; - List cols = - List.of( - new ColSpec("c_int", "INT", "1000", "1000", ColumnTypeName.INT, "int", "\"integer\""), - new ColSpec( - "c_long", "BIGINT", "100000", "100000", ColumnTypeName.LONG, "bigint", "\"long\""), - new ColSpec( - "c_float", "FLOAT", "2.5", "2.5", ColumnTypeName.FLOAT, "float", "\"float\""), - new ColSpec( - "c_double", "DOUBLE", "1.5", "1.5", ColumnTypeName.DOUBLE, "double", "\"double\""), - new ColSpec( - "c_decimal", - "DECIMAL(10,2)", - "123.45", - "123.45", - ColumnTypeName.DECIMAL, - "decimal(10,2)", - "\"decimal(10,2)\""), - new ColSpec( - "c_string", "STRING", "'test'", "test", ColumnTypeName.STRING, "string", "\"string\""), - // BINARY round-trips as a Java byte[], not a String, so rowValue is unused; the check - // below compares the raw bytes against the inserted X'CAFEBABE' instead. - new ColSpec( - "c_binary", - "BINARY", - "X'CAFEBABE'", - null, - ColumnTypeName.BINARY, - "binary", - "\"binary\""), - new ColSpec( - "c_boolean", - "BOOLEAN", - "true", - "true", - ColumnTypeName.BOOLEAN, - "boolean", - "\"boolean\""), - new ColSpec( - "c_date", - "DATE", - "DATE'2025-01-01'", - "2025-01-01", - ColumnTypeName.DATE, - "date", - "\"date\""), - new ColSpec( - "c_timestamp", - "TIMESTAMP", - "TIMESTAMP'2025-01-01 12:00:00'", - "2025-01-01 12:00:00.0", - ColumnTypeName.TIMESTAMP, - "timestamp", - "\"timestamp\""), - new ColSpec( - "c_timestamp_ntz", - "TIMESTAMP_NTZ", - "TIMESTAMP_NTZ'2025-01-01 12:00:00'", - "2025-01-01T12:00", - ColumnTypeName.TIMESTAMP_NTZ, - "timestamp_ntz", - "\"timestamp_ntz\""), - new ColSpec( - "c_arr", - "ARRAY", - "array(1, 2, 3)", - "ArraySeq(1, 2, 3)", - ColumnTypeName.ARRAY, - "array", - arrJson), - new ColSpec( - "c_map", - "MAP", - "map('k', 10)", - "Map(k -> 10)", - ColumnTypeName.MAP, - "map", - mapJson), - new ColSpec( - "c_struct", - "STRUCT", - "struct(42, 'x')", - "[42,x]", - ColumnTypeName.STRUCT, - "struct", - structJson)); - - session = createSparkSessionWithCatalogs(CATALOG_NAME); - String tableName = TEST_TABLE + "_types"; - String fullTableName = - setupTable( - new TableSetupOptions() - .setCatalogName(CATALOG_NAME) - .setTableName(tableName) - .setColumns( - cols.stream() - .map(c -> Pair.of(c.name, c.sqlType)) - .collect(Collectors.toList()))); - - String colNames = cols.stream().map(c -> c.name).collect(Collectors.joining(", ")); - sql( - "INSERT INTO %s (%s) VALUES (%s)", - fullTableName, - colNames, - cols.stream().map(c -> c.insertValue).collect(Collectors.joining(", "))); - - List queryResult = sql("SELECT %s FROM %s", colNames, fullTableName); - assertThat(queryResult).hasSize(1); - Row row = queryResult.get(0); - for (int i = 0; i < cols.size(); i++) { - ColSpec spec = cols.get(i); - if (spec.typeName == ColumnTypeName.BINARY) { - assertCafebabeBinary(row.get(i), spec.name); - } else { - assertThat(row.get(i).toString()) - .as("row value for %s", spec.name) - .isEqualTo(spec.rowValue); - } - } - - TableInfo tableInfo = tableOperations.getTable(fullTableName); - // A client-supplied LOCATION registers the table EXTERNAL, otherwise MANAGED; the format is - // always ICEBERG. Confirms the managed/external subclasses produce what they claim. - assertThat(tableInfo.getTableType()) - .isEqualTo(isManagedTable() ? TableType.MANAGED : TableType.EXTERNAL); - assertThat(tableInfo.getDataSourceFormat()).isEqualTo(DataSourceFormat.ICEBERG); - List columns = tableInfo.getColumns(); - assertThat(columns).hasSize(cols.size()); - for (int i = 0; i < cols.size(); i++) { - ColumnInfo col = columns.get(i); - ColSpec spec = cols.get(i); - assertThat(col.getName()).as("name[%d]", i).isEqualTo(spec.name); - assertThat(col.getTypeName()).as("typeName for %s", spec.name).isEqualTo(spec.typeName); - assertThat(col.getTypeText()).as("typeText for %s", spec.name).isEqualTo(spec.typeText); - assertThat(col.getTypeJson()) - .as("typeJson for %s", spec.name) - .isEqualTo(structFieldTypeJson(spec.name, spec.dataTypeJson)); - } - } } diff --git a/project/CrossSparkVersions.scala b/project/CrossSparkVersions.scala index 790d076403..705f4d6d6f 100644 --- a/project/CrossSparkVersions.scala +++ b/project/CrossSparkVersions.scala @@ -28,6 +28,10 @@ case class SparkVersionSpec( // identical shims are written once instead of copied per version. sbt ignores non-existent // dirs, so listing a dir a module doesn't use is harmless. additionalSourceDirs: Seq[String], + // Whether an iceberg-spark-runtime is published for this Spark version. Declared per version + // rather than inferred from the version number: Iceberg's Spark runtime lags new Spark releases, + // so a version-math guess would silently break when a runtime is (or isn't) published. + supportIceberg: Boolean = false, requiresSparkCommit: Boolean = false, sourceBuildArtifactBaseVersion: Option[String] = None, sourceBuildDefaultRef: Option[String] = None @@ -67,11 +71,13 @@ object SparkVersionSpec { case l: List[_] => l.map(_.asInstanceOf[String]) } SparkVersionSpec( - v("version").asInstanceOf[String], - sourceDirs, - v.get("requiresSparkCommit").exists(_.asInstanceOf[Boolean]), - v.get("sourceBuildArtifactBaseVersion").map(_.asInstanceOf[String]), - v.get("sourceBuildDefaultRef").map(_.asInstanceOf[String]) + fullVersion = v("version").asInstanceOf[String], + additionalSourceDirs = sourceDirs, + supportIceberg = v.get("supportIceberg").exists(_.asInstanceOf[Boolean]), + requiresSparkCommit = v.get("requiresSparkCommit").exists(_.asInstanceOf[Boolean]), + sourceBuildArtifactBaseVersion = + v.get("sourceBuildArtifactBaseVersion").map(_.asInstanceOf[String]), + sourceBuildDefaultRef = v.get("sourceBuildDefaultRef").map(_.asInstanceOf[String]) ) } (default, versions) @@ -172,6 +178,37 @@ object CrossSparkVersions extends AutoPlugin { else s"${baseName}_${getSparkVersionSpec().shortVersion}" } + /** + * Test-only `iceberg-spark-runtime` for the Spark + Iceberg suites; empty when `supportIceberg` is + * false. Those suites live in a shim dir, so a version with `supportIceberg` must also list that + * dir in its `sourceDirs`, or the dep is added but no Iceberg test is compiled. + * + * The runtime does not bundle the AWS SDK; the fake-S3 Iceberg tests get it from the + * server-shaded test jar (the server depends on it), which precedes this runtime on the test + * classpath. + */ + def icebergSparkTestDeps(icebergVersion: String): Seq[ModuleID] = { + val spec = getSparkVersionSpec() + if (!spec.supportIceberg) Seq.empty + else + Seq( + "org.apache.iceberg" % + s"iceberg-spark-runtime-${spec.shortVersion}_2.13" % icebergVersion % Test) + } + + /** + * Test-only `delta-spark` matching the active Spark version. `delta-spark` is published for every + * supported Spark version, so this is unconditional except for one escape hatch: when UC is + * published to local Maven before Delta is built (the CI pre-Delta publishM2 step), the matching + * Delta artifact may not exist yet, so `-DskipDeltaSpark=true` drops it to avoid a resolution + * failure. + */ + def deltaSparkTestDeps(deltaVersion: String): Seq[ModuleID] = { + if (sys.props.getOrElse("skipDeltaSpark", "false").toBoolean) Seq.empty + else + Seq("io.delta" %% s"delta-spark_${getSparkVersionSpec().shortVersion}" % deltaVersion % Test) + } + /** * Settings for Spark-dependent modules. Includes: * - sparkVersion setting (queryable via `show spark/sparkVersion`) diff --git a/project/spark-versions.json b/project/spark-versions.json index ca5376cc96..0f5612e178 100644 --- a/project/spark-versions.json +++ b/project/spark-versions.json @@ -1,8 +1,8 @@ { "default": "4.1.0", "versions": [ - {"version": "4.0.0", "sourceDirs": ["scala-shims/spark-4.0-4.1"]}, - {"version": "4.1.0", "sourceDirs": ["scala-shims/spark-4.0-4.1"]}, - {"version": "4.2.0", "sourceDirs": ["scala-shims/spark-4.2"]} + {"version": "4.0.0", "sourceDirs": ["scala-shims/spark-4.0-4.1"], "supportIceberg": true}, + {"version": "4.1.0", "sourceDirs": ["scala-shims/spark-4.0-4.1"], "supportIceberg": true}, + {"version": "4.2.0", "sourceDirs": ["scala-shims/spark-4.2"], "supportIceberg": false} ] }