diff --git a/build.sbt b/build.sbt index ac5e082f88..9728f63322 100644 --- a/build.sbt +++ b/build.sbt @@ -26,14 +26,6 @@ lazy val deltaVersion = sys.props.getOrElse("deltaVersion", "4.3.1") // This String val is used for libraryDependencies coordinates; the SettingKey is // queryable in SBT via `show spark/sparkVersion`. lazy val sparkVersion = CrossSparkVersions.getSparkArtifactVersion() -lazy val sparkMajorMinorVersion = CrossSparkVersions.getSparkVersionSpec().shortVersion - -// delta-spark is only needed for tests. When UC is published to local Maven before -// Delta is built (e.g. CI pre-Delta publishM2 step), the matching Delta artifact may -// not exist yet. Pass -DskipDeltaSpark=true to exclude it and avoid resolution failures. -def deltaSparkTestDeps: Seq[ModuleID] = - if (sys.props.getOrElse("skipDeltaSpark", "false").toBoolean) Seq.empty - else Seq("io.delta" %% s"delta-spark_$sparkMajorMinorVersion" % deltaVersion % Test) // Apache Snapshots resolver is in build/sbt-config/repositories (global). // No per-module sparkResolvers needed. @@ -695,6 +687,8 @@ lazy val spark = (project in file("connectors/spark")) "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", ), + // Arrow-backed vectorized reads need java.nio opened for off-heap memory access. + Test / javaOptions += "--add-opens=java.base/java.nio=ALL-UNNAMED", javafmtCheckSettings(), javaCheckstyleSettings("dev/checkstyle-config.xml"), Compile / compile / javacOptions ++= javacRelease11, @@ -743,7 +737,8 @@ lazy val spark = (project in file("connectors/spark")) "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % Test, "org.projectlombok" % "lombok" % "1.18.32" % Test, "com.google.cloud.bigdataoss" % "gcs-connector" % "3.0.2" % Test classifier "shaded", - ) ++ deltaSparkTestDeps, + ) ++ CrossSparkVersions.deltaSparkTestDeps(deltaVersion) + ++ CrossSparkVersions.icebergSparkTestDeps(icebergVersion), dependencyOverrides ++= Seq( "com.fasterxml.jackson.core" % "jackson-databind" % sparkJacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkJacksonVersion, @@ -833,7 +828,7 @@ lazy val integrationTests = (project in file("integration-tests")) "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % Test, "org.apache.hadoop" % "hadoop-azure" % hadoopVersion % Test, "com.google.cloud.bigdataoss" % "gcs-connector" % "3.0.2" % Test classifier "shaded", - ) ++ deltaSparkTestDeps, + ) ++ CrossSparkVersions.deltaSparkTestDeps(deltaVersion), dependencyOverrides ++= Seq( "com.fasterxml.jackson.core" % "jackson-databind" % sparkJacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkJacksonVersion, diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java index 5023854cd6..c8191e1200 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseSparkIntegrationTest.java @@ -32,6 +32,9 @@ public abstract class BaseSparkIntegrationTest extends BaseCRUDTest { protected ArrayList createdCatalogs = new ArrayList<>(); protected static final String SPARK_CATALOG = "spark_catalog"; + /** The server's Iceberg REST catalog path, served under the UC base path. */ + private static final String ICEBERG_REST_PATH = "/api/2.1/unity-catalog/iceberg"; + /** S3 bucket with no credentials configured on server - for testing SSP fallback. */ public static final String NO_CREDS_BUCKET = "test-bucket-2-no-creds"; @@ -39,6 +42,17 @@ public abstract class BaseSparkIntegrationTest extends BaseCRUDTest { // Each test would create this session. It will be closed automatically. protected SparkSession session; + /** + * True when this suite drives the given catalog as Iceberg's own REST {@code SparkCatalog} + * against the server's Iceberg REST catalog, rather than through the UC Spark connector (which + * has no Iceberg support). Per catalog, so one session can mix both (e.g. a Delta join against + * Iceberg); subclasses flip this, and {@link #createSparkSessionWithCatalogs} is the only + * consumer, so no subclass needs to re-implement session creation. + */ + protected boolean isIcebergCatalog(String catalog) { + return false; + } + private void createCommonResources() throws ApiException { // Common setup operations such as creating a catalog and schema catalogOperations.createCatalog( @@ -52,24 +66,45 @@ protected SparkSession createSparkSessionWithCatalogs(String... catalogs) { return createSparkSessionWithCatalogs(true, true, catalogs); } + /** The base Spark builder shared by all integration tests (local master, small shuffle width). */ + private SparkSession.Builder newSparkSessionBuilder() { + return SparkSession.builder() + .appName("test") + .master("local[*]") + .config("spark.sql.shuffle.partitions", "4"); + } + protected SparkSession createSparkSessionWithCatalogs( boolean renewCred, boolean credScopedFsEnabled, String... catalogs) { + // renewCred / credScopedFsEnabled are UC-connector options with no Iceberg REST analog. + // A catalog is wired either as an Iceberg REST catalog or through the UC connector + // (UCSingleCatalog), which serves Delta, Parquet, and the other non-Iceberg formats. + boolean anyIceberg = false; + boolean anyUcConnector = false; + for (String catalog : catalogs) { + if (isIcebergCatalog(catalog)) { + anyIceberg = true; + } else { + anyUcConnector = true; + } + } + // Both extensions can coexist in one session, which is what lets a single query join a Delta + // table against an Iceberg table. The UC connector carries Delta support, so its path loads the + // Delta extension (harmless for the non-Delta formats it also serves, e.g. Parquet). + List extensions = new ArrayList<>(); + if (anyUcConnector) { + extensions.add("io.delta.sql.DeltaSparkSessionExtension"); + } + if (anyIceberg) { + extensions.add("org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"); + } SparkSession.Builder builder = - SparkSession.builder() - .appName("test") - .master("local[*]") - .config("spark.sql.shuffle.partitions", "4") - .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension"); + newSparkSessionBuilder().config("spark.sql.extensions", String.join(",", extensions)); for (String catalog : catalogs) { - String catalogConf = "spark.sql.catalog." + catalog; builder = - builder - .config(catalogConf, UCSingleCatalog.class.getName()) - .config(catalogConf + "." + OptionsUtil.URI, serverConfig.getServerUrl()) - .config(catalogConf + "." + OptionsUtil.TOKEN, serverConfig.getAuthToken()) - .config(catalogConf + "." + OptionsUtil.WAREHOUSE, catalog) - .config(catalogConf + "." + OptionsUtil.RENEW_CREDENTIAL_ENABLED, renewCred) - .config(catalogConf + "." + OptionsUtil.CRED_SCOPED_FS_ENABLED, credScopedFsEnabled); + isIcebergCatalog(catalog) + ? icebergCatalogConfig(builder, catalog) + : ucSingleCatalogConfig(builder, catalog, renewCred, credScopedFsEnabled); if (!List.of(SPARK_CATALOG, CATALOG_NAME).contains(catalog)) { createTestCatalog(catalog); } @@ -78,9 +113,12 @@ protected SparkSession createSparkSessionWithCatalogs( // (DeltaLog.checkRequiredConfigurations); otherwise a Delta table -- even in a named UC catalog // -- fails with DELTA_CONFIGURE_SPARK_SESSION_WITH_EXTENSION_AND_CATALOG. When the caller did // not configure spark_catalog above, point it at Delta's own catalog to satisfy that check. - if (!List.of(catalogs).contains(SPARK_CATALOG)) { + // This applies to any UC-connector session (the connector may serve Delta); Iceberg has no such + // requirement and these suites address tables only through the named UC catalog, so a + // pure-Iceberg session needs no spark_catalog override and leaves it as the built-in one. + if (anyUcConnector && !List.of(catalogs).contains(SPARK_CATALOG)) { builder.config( - "spark.sql.catalog." + SPARK_CATALOG, "org.apache.spark.sql.delta.catalog.DeltaCatalog"); + catalogConfKey(SPARK_CATALOG), "org.apache.spark.sql.delta.catalog.DeltaCatalog"); } // Use fake file system for cloud storage so that we can test credentials. builder.config("spark.hadoop.fs.s3.impl", S3CredentialTestFileSystem.class.getName()); @@ -89,6 +127,43 @@ protected SparkSession createSparkSessionWithCatalogs( return builder.getOrCreate(); } + /** The Spark SQL config key prefix for a catalog, e.g. {@code spark.sql.catalog.}. */ + private static String catalogConfKey(String catalog) { + return "spark.sql.catalog." + catalog; + } + + /** Wires one catalog through the UC Spark connector ({@link UCSingleCatalog}), the Delta path. */ + private SparkSession.Builder ucSingleCatalogConfig( + SparkSession.Builder builder, + String catalog, + boolean renewCred, + boolean credScopedFsEnabled) { + String catalogConf = catalogConfKey(catalog); + return builder + .config(catalogConf, UCSingleCatalog.class.getName()) + .config(catalogConf + "." + OptionsUtil.URI, serverConfig.getServerUrl()) + .config(catalogConf + "." + OptionsUtil.TOKEN, serverConfig.getAuthToken()) + .config(catalogConf + "." + OptionsUtil.WAREHOUSE, catalog) + .config(catalogConf + "." + OptionsUtil.RENEW_CREDENTIAL_ENABLED, renewCred) + .config(catalogConf + "." + OptionsUtil.CRED_SCOPED_FS_ENABLED, credScopedFsEnabled); + } + + /** + * Wires one catalog as Iceberg's REST {@code SparkCatalog} against the server's Iceberg REST + * catalog, reading and writing data files through Iceberg's {@code HadoopFileIO} on the local + * warehouse. + */ + private SparkSession.Builder icebergCatalogConfig(SparkSession.Builder builder, String catalog) { + String catalogConf = catalogConfKey(catalog); + return builder + .config(catalogConf, "org.apache.iceberg.spark.SparkCatalog") + .config(catalogConf + ".type", "rest") + .config(catalogConf + ".uri", serverConfig.getServerUrl() + ICEBERG_REST_PATH) + .config(catalogConf + ".warehouse", catalog) + .config(catalogConf + ".io-impl", "org.apache.iceberg.hadoop.HadoopFileIO") + .config(catalogConf + ".token", serverConfig.getAuthToken()); + } + protected List sql(String statement, Object... args) { return session.sql(String.format(statement, args)).collectAsList(); } diff --git a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java index 766f582fb3..3328168768 100644 --- a/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java +++ b/connectors/spark/src/test/java/io/unitycatalog/spark/BaseTableReadWriteTest.java @@ -10,7 +10,9 @@ import io.unitycatalog.client.ApiException; import io.unitycatalog.client.model.ColumnInfo; import io.unitycatalog.client.model.ColumnTypeName; +import io.unitycatalog.client.model.DataSourceFormat; import io.unitycatalog.client.model.TableInfo; +import io.unitycatalog.client.model.TableType; import io.unitycatalog.server.base.table.TableOperations; import io.unitycatalog.server.persist.utils.PagedListingHelper; import io.unitycatalog.server.sdk.tables.SdkTableOperations; @@ -29,6 +31,7 @@ import org.apache.spark.sql.Row; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.StructType; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIf; @@ -243,6 +246,10 @@ protected final boolean testingDelta() { return tableFormat().equalsIgnoreCase("DELTA"); } + protected final boolean testingIceberg() { + return tableFormat().equalsIgnoreCase("ICEBERG"); + } + protected abstract boolean isManagedTable(); protected final boolean canUpdateColumnsToUC() { @@ -434,12 +441,15 @@ public void testTimeTravelDeltaTable() { @MethodSource("cloudParameters") public void testTableOperations( String scheme, boolean renewCredEnabled, boolean credScopedFsEnabled) { + // t1 goes in the first session catalog: spark_catalog for the UC connector, or the named + // catalog when that is the only one wired (Iceberg), so both tables then share it. + List catalogs = sessionCatalogNames(); session = createSparkSessionWithCatalogs( - renewCredEnabled, credScopedFsEnabled, SPARK_CATALOG, CATALOG_NAME); + renewCredEnabled, credScopedFsEnabled, catalogs.toArray(new String[0])); // t1 has (1, 'a') - String t1 = setupTable(scheme, SPARK_CATALOG, TEST_TABLE); + String t1 = setupTable(scheme, catalogs.get(0), TEST_TABLE); testTableReadWrite(t1); // t2 has (2, 'a') @@ -447,8 +457,9 @@ public void testTableOperations( sql("INSERT INTO %s SELECT 2, 'a'", t2); validateRows(sql("SELECT * FROM %s", t2), Pair.of(2, "a")); - if (testingDelta()) { - // UPDATE, MERGE INTO and DELETE are only supported in Delta tables. + if (testingDelta() || testingIceberg()) { + // UPDATE, MERGE INTO and DELETE need a format with row-level support (Delta or Iceberg); the + // external-Parquet path does not support them. // Test UPDATE. The table t2 will have (2, 'b') sql("UPDATE %s SET s = 'b' WHERE i = 2", t2); @@ -464,18 +475,29 @@ public void testTableOperations( } // Test SHOW TABLES - List tables1 = sql("SHOW TABLES in %s.%s", SPARK_CATALOG, SCHEMA_NAME); - assertThat(tables1).hasSize(1); - assertThat(tables1.get(0).getString(0)).isEqualTo(SCHEMA_NAME); - assertThat(tables1.get(0).getString(1)).isEqualTo(TEST_TABLE); - List tables2 = sql("SHOW TABLES in %s.%s", CATALOG_NAME, SCHEMA_NAME); - assertThat(tables2).hasSize(1); - assertThat(tables2.get(0).getString(0)).isEqualTo(SCHEMA_NAME); - assertThat(tables2.get(0).getString(1)).isEqualTo(ANOTHER_TEST_TABLE); - - assertThatThrownBy(() -> sql("SHOW TABLES in a.b.c")) - .isInstanceOf(ApiException.class) - .hasMessageContaining("Nested namespaces are not supported"); + if (testingIceberg()) { + // Both tables live in the single wired catalog. + List tableNames = + sql("SHOW TABLES in %s.%s", CATALOG_NAME, SCHEMA_NAME).stream() + .map(row -> row.getString(1)) + .collect(Collectors.toList()); + assertThat(tableNames).containsExactlyInAnyOrder(TEST_TABLE, ANOTHER_TEST_TABLE); + // A multi-level namespace does not exist as a UC schema, so listing under it is rejected. + assertIcebergSchemaNotFound(() -> sql("SHOW TABLES in %s.a.b", CATALOG_NAME)); + } else { + List tables1 = sql("SHOW TABLES in %s.%s", SPARK_CATALOG, SCHEMA_NAME); + assertThat(tables1).hasSize(1); + assertThat(tables1.get(0).getString(0)).isEqualTo(SCHEMA_NAME); + assertThat(tables1.get(0).getString(1)).isEqualTo(TEST_TABLE); + List tables2 = sql("SHOW TABLES in %s.%s", CATALOG_NAME, SCHEMA_NAME); + assertThat(tables2).hasSize(1); + assertThat(tables2.get(0).getString(0)).isEqualTo(SCHEMA_NAME); + assertThat(tables2.get(0).getString(1)).isEqualTo(ANOTHER_TEST_TABLE); + + assertThatThrownBy(() -> sql("SHOW TABLES in a.b.c")) + .isInstanceOf(ApiException.class) + .hasMessageContaining("Nested namespaces are not supported"); + } // DROP TABLE for (String fullTableName : List.of(t1, t2)) { @@ -483,35 +505,42 @@ public void testTableOperations( sql("DROP TABLE %s", fullTableName); assertThat(session.catalog().tableExists(fullTableName)).isFalse(); } - // Delta < 4.3.0 forwards the 4-part identifier to UC, which rejects it server-side with - // ApiException("Nested namespaces are not supported"). Delta >= 4.3.0 ships - // UCDeltaCatalogClientImpl, which rejects the same input client-side with - // IllegalArgumentException("UC table identifier must be one of ...") before reaching UC. - boolean rejectedClientSide = - DeltaVersionUtils.isDeltaAtLeast(MIN_DELTA_VERSION_FOR_UC_DELTA_API); - Class expectedType = - rejectedClientSide ? IllegalArgumentException.class : ApiException.class; - String expectedMessage = - rejectedClientSide - ? "UC table identifier must be one of" - : "Nested namespaces are not supported"; - assertThatThrownBy(() -> sql("DROP TABLE a.b.c.d")) - .isInstanceOf(expectedType) - .hasMessageContaining(expectedMessage); + // Dropping a malformed multi-level table identifier is rejected. + if (testingIceberg()) { + // The namespace a.b does not exist as a UC schema, so the drop is rejected before the table. + assertIcebergSchemaNotFound(() -> sql("DROP TABLE %s.a.b.c", CATALOG_NAME)); + } else { + // Delta < 4.3.0 forwards the 4-part identifier to UC, which rejects it server-side with + // ApiException("Nested namespaces are not supported"). Delta >= 4.3.0 ships + // UCDeltaCatalogClientImpl, which rejects the same input client-side with + // IllegalArgumentException("UC table identifier must be one of ...") before reaching UC. + boolean rejectedClientSide = + DeltaVersionUtils.isDeltaAtLeast(MIN_DELTA_VERSION_FOR_UC_DELTA_API); + Class expectedType = + rejectedClientSide ? IllegalArgumentException.class : ApiException.class; + String expectedMessage = + rejectedClientSide + ? "UC table identifier must be one of" + : "Nested namespaces are not supported"; + assertThatThrownBy(() -> sql("DROP TABLE a.b.c.d")) + .isInstanceOf(expectedType) + .hasMessageContaining(expectedMessage); + } } // With page size 2 and 3 tables, pagination must occur (2 API calls) for all 3 to be returned. // Asserting all 3 names proves pagination worked end-to-end. @Test public void testListTablesPagination() { - session = createSparkSessionWithCatalogs(SPARK_CATALOG); + String catalog = sessionCatalogNames().get(0); + session = createSparkSessionWithCatalogs(catalog); Integer originalPageSize = PagedListingHelper.DEFAULT_PAGE_SIZE; try { PagedListingHelper.DEFAULT_PAGE_SIZE = 2; - setupTable(SPARK_CATALOG, "pagination_table_1"); - setupTable(SPARK_CATALOG, "pagination_table_2"); - setupTable(SPARK_CATALOG, "pagination_table_3"); - List tables = sql("SHOW TABLES in %s.%s", SPARK_CATALOG, SCHEMA_NAME); + setupTable(catalog, "pagination_table_1"); + setupTable(catalog, "pagination_table_2"); + setupTable(catalog, "pagination_table_3"); + List tables = sql("SHOW TABLES in %s.%s", catalog, SCHEMA_NAME); assertThat(tables).hasSize(3); List tableNames = tables.stream().map(row -> row.getString(1)).sorted().collect(Collectors.toList()); @@ -519,9 +548,9 @@ public void testListTablesPagination() { .containsExactly("pagination_table_1", "pagination_table_2", "pagination_table_3"); } finally { PagedListingHelper.DEFAULT_PAGE_SIZE = originalPageSize; - sql("DROP TABLE IF EXISTS %s.%s.pagination_table_1", SPARK_CATALOG, SCHEMA_NAME); - sql("DROP TABLE IF EXISTS %s.%s.pagination_table_2", SPARK_CATALOG, SCHEMA_NAME); - sql("DROP TABLE IF EXISTS %s.%s.pagination_table_3", SPARK_CATALOG, SCHEMA_NAME); + sql("DROP TABLE IF EXISTS %s.%s.pagination_table_1", catalog, SCHEMA_NAME); + sql("DROP TABLE IF EXISTS %s.%s.pagination_table_2", catalog, SCHEMA_NAME); + sql("DROP TABLE IF EXISTS %s.%s.pagination_table_3", catalog, SCHEMA_NAME); } } @@ -534,7 +563,10 @@ public void testHyphenInTableName() { String catalogName = "test-catalog-name"; String schemaName = "test-schema-name"; String tableName = "test-table-name"; - session = createSparkSessionWithCatalogs(SPARK_CATALOG, catalogName); + session = + testingIceberg() + ? createSparkSessionWithCatalogs(catalogName) + : createSparkSessionWithCatalogs(SPARK_CATALOG, catalogName); sql("CREATE SCHEMA `%s`.`%s`", catalogName, schemaName); String fullTableName = setupTable( @@ -557,15 +589,15 @@ public void testHyphenInTableName() { /** * Specification for one column in testTableWithSupportedDataTypes: its SQL DDL type, the SQL - * literal used in the INSERT, the expected toString() from the result row (null = byte-array ref - * check via startsWith("[B@")), and the expected UC catalog metadata fields. + * literal used in the INSERT, the expected toString() from the result row (null for BINARY, whose + * raw bytes are compared directly), and the expected UC catalog metadata fields. */ @Getter private static final class ColSpec { private final String name; private final String sqlType; private final String insertValue; - private final String rowValue; // null = byte-array object ref, checked with startsWith + private final String rowValue; // null for BINARY, whose bytes are compared directly private final ColumnTypeName typeName; private final String typeText; private final boolean nullable; @@ -603,10 +635,6 @@ private ColSpec( } } - private static String structFieldTypeJson(String name, String dataTypeJson) { - return structFieldTypeJson(name, dataTypeJson, true, "{}"); - } - private static String structFieldTypeJson( String name, String dataTypeJson, boolean nullable, String metadataJson) { return String.format( @@ -821,10 +849,23 @@ public void testTableWithSupportedDataTypes() throws ApiException { .filter(col -> !col.getName().equals("col_variant")) .collect(Collectors.toList()); } + if (testingIceberg()) { + // Iceberg has no distinct TINYINT / SMALLINT (widened to int) or CHAR / VARCHAR (stored as + // string), and VARIANT needs format v3, so these would not round-trip as declared. + List unsupported = + List.of("col_tinyint", "col_smallint", "col_char", "col_varchar", "col_variant"); + cols = + cols.stream() + .filter(col -> !unsupported.contains(col.getName())) + .collect(Collectors.toList()); + } - session = createSparkSessionWithCatalogs(SPARK_CATALOG, CATALOG_NAME); + session = createSparkSessionWithCatalogs(sessionCatalogNames().toArray(new String[0])); String tableName = TEST_TABLE + "_complex_type"; - List partitionColumns = List.of("col_string", "col_bigint"); + // UC records no partitionIndex for Iceberg (its partition spec is not per-column), so Iceberg + // tables are created unpartitioned. + List partitionColumns = + testingIceberg() ? List.of() : List.of("col_string", "col_bigint"); String fullTableName = setupTable( new TableSetupOptions() @@ -845,13 +886,19 @@ public void testTableWithSupportedDataTypes() throws ApiException { List queryResult = sql("SELECT %s FROM %s", colNames, fullTableName); assertThat(queryResult).hasSize(1); + Row resultRow = queryResult.get(0); List row = - IntStream.range(0, queryResult.get(0).length()) - .mapToObj( - i -> queryResult.get(0).isNullAt(i) ? null : queryResult.get(0).get(i).toString()) + IntStream.range(0, resultRow.length()) + .mapToObj(i -> resultRow.isNullAt(i) ? null : resultRow.get(i).toString()) .collect(Collectors.toList()); TableInfo tableInfo = tableOperations.getTable(fullTableName); + // A client-supplied LOCATION registers the table EXTERNAL, otherwise MANAGED; the format is the + // suite's tableFormat(). Confirms each managed/external subclass produces what it claims. + assertThat(tableInfo.getTableType()) + .isEqualTo(isManagedTable() ? TableType.MANAGED : TableType.EXTERNAL); + assertThat(tableInfo.getDataSourceFormat()) + .isEqualTo(DataSourceFormat.valueOf(tableFormat().toUpperCase())); List columns = tableInfo.getColumns(); assertThat(columns).hasSize(cols.size()); @@ -873,8 +920,8 @@ public void testTableWithSupportedDataTypes() throws ApiException { for (int i = 0; i < cols.size(); i++) { ColSpec spec = cols.get(i); if (spec.getTypeName() == ColumnTypeName.BINARY) { - // BINARY: row value is a Java byte array — toString() produces a ref like "[B@..." - assertThat(row.get(i)).as("row value for %s", spec.getName()).startsWith("[B@"); + // Read BINARY from the raw row; stringified `row` only keeps its "[B@" ref. + assertCafebabeBinary(resultRow.get(i), spec.getName()); } else { assertThat(row.get(i)).as("row value for %s", spec.getName()).isEqualTo(spec.getRowValue()); } @@ -944,4 +991,30 @@ protected void testTableReadWriteCreatedAsSelect( sql("INSERT INTO %s SELECT %d, '%s'", tableFullName, newValue.getLeft(), newValue.getRight()); validateRows(sql("SELECT * FROM %s ORDER BY i", tableFullName), asSelect, newValue); } + + /** + * Asserts {@code op} is rejected specifically because its multi-level namespace does not exist as + * a UC schema. The Iceberg REST client maps UC's rejection to {@code NoSuchNamespaceException}; + * checked by class name (not an imported type, so this shared base still compiles on Spark 4.2, + * which has no Iceberg on its classpath) and by message, so an unrelated non-2xx (a 500 or auth + * failure, which also surfaces as a {@code RESTException}) does not make the assertion pass. + */ + private static void assertIcebergSchemaNotFound(ThrowingCallable op) { + assertThatThrownBy(op) + .hasMessageContaining("Schema not found") + .satisfies( + t -> + assertThat(t.getClass().getName()) + .isEqualTo("org.apache.iceberg.exceptions.NoSuchNamespaceException")); + } + + /** + * Asserts a BINARY column round-tripped the inserted {@code X'CAFEBABE'} byte-for-byte; {@code + * rowValue} is the raw row cell, which is a Java {@code byte[]} for BINARY. + */ + private static void assertCafebabeBinary(Object rowValue, String columnName) { + assertThat((byte[]) rowValue) + .as("row value for %s", columnName) + .containsExactly(0xCA, 0xFE, 0xBA, 0xBE); + } } diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java new file mode 100644 index 0000000000..2a17cbf776 --- /dev/null +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/DeltaIcebergCrossFormatJoinTest.java @@ -0,0 +1,87 @@ +package io.unitycatalog.spark; + +import static io.unitycatalog.server.utils.TestUtils.CATALOG_NAME; +import static io.unitycatalog.server.utils.TestUtils.SCHEMA_NAME; +import static io.unitycatalog.server.utils.TestUtils.createApiClient; +import static org.assertj.core.api.Assertions.assertThat; + +import io.unitycatalog.client.ApiException; +import io.unitycatalog.client.model.DataSourceFormat; +import io.unitycatalog.server.base.table.TableOperations; +import io.unitycatalog.server.sdk.tables.SdkTableOperations; +import io.unitycatalog.server.utils.ServerProperties; +import java.util.List; +import org.apache.spark.sql.Row; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Verifies that a single SparkSession can be configured with both a Delta-capable catalog (the UC + * connector, {@link UCSingleCatalog}) and an Iceberg catalog (Iceberg's own REST {@code + * SparkCatalog} against UC's Iceberg REST endpoint) at the same time, and can join a Delta table + * against an Iceberg table across the two catalogs. + * + *

Both catalogs point at the same UC server: the Delta table lives in the {@code spark_catalog} + * UC catalog (reached through the connector, which also satisfies Delta's required-catalog check) + * and the Iceberg table in {@code CATALOG_NAME} (reached through the Iceberg REST client). The + * shared {@code createSparkSessionWithCatalogs} derives both session extensions from the catalog + * mix, and storage is a local {@code file://} warehouse. + * + *

Lives under {@code src/test/scala-shims/spark-4.0-4.1} so it is compiled only for the Spark + * versions that have an Iceberg Spark runtime ({@code supportIceberg} in the cross-Spark build). + */ +public class DeltaIcebergCrossFormatJoinTest extends BaseSparkIntegrationTest { + + private TableOperations tableOperations; + + @BeforeEach + @Override + public void setUp() { + super.setUp(); + tableOperations = new SdkTableOperations(createApiClient(serverConfig)); + } + + @Override + protected void setUpProperties() { + super.setUpProperties(); + serverProperties.setProperty(ServerProperties.Property.ICEBERG_TABLE_ENABLED.getKey(), "true"); + } + + // Only CATALOG_NAME is Iceberg; SPARK_CATALOG stays on the UC connector for Delta, so the base + // wires a mixed session with both extensions. + @Override + protected boolean isIcebergCatalog(String catalog) { + return catalog.equals(CATALOG_NAME); + } + + @Test + public void testJoinDeltaAndIcebergTables() throws ApiException { + session = createSparkSessionWithCatalogs(false, false, SPARK_CATALOG, CATALOG_NAME); + + String deltaTable = String.join(".", SPARK_CATALOG, SCHEMA_NAME, "delta_events"); + String icebergTable = String.join(".", CATALOG_NAME, SCHEMA_NAME, "iceberg_events"); + + sql("CREATE TABLE %s (id INT, delta_val STRING) USING delta", deltaTable); + sql("INSERT INTO %s VALUES (1, 'delta-one'), (2, 'delta-two')", deltaTable); + + sql("CREATE TABLE %s (id INT, iceberg_val STRING) USING iceberg", icebergTable); + sql("INSERT INTO %s VALUES (1, 'iceberg-one'), (3, 'iceberg-three')", icebergTable); + + // The two tables are genuinely different formats in UC. + assertThat(tableOperations.getTable(deltaTable).getDataSourceFormat()) + .isEqualTo(DataSourceFormat.DELTA); + assertThat(tableOperations.getTable(icebergTable).getDataSourceFormat()) + .isEqualTo(DataSourceFormat.ICEBERG); + + // Cross-format inner join on id: only id = 1 exists in both tables. + List joined = + sql( + "SELECT d.id, d.delta_val, i.iceberg_val FROM %s d JOIN %s i ON d.id = i.id", + deltaTable, icebergTable); + assertThat(joined).hasSize(1); + Row row = joined.get(0); + assertThat(row.getInt(0)).isEqualTo(1); + assertThat(row.getString(1)).isEqualTo("delta-one"); + assertThat(row.getString(2)).isEqualTo("iceberg-one"); + } +} diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergExternalTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergExternalTableReadWriteTest.java new file mode 100644 index 0000000000..b48b71a67c --- /dev/null +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergExternalTableReadWriteTest.java @@ -0,0 +1,37 @@ +package io.unitycatalog.spark; + +import java.io.File; +import lombok.SneakyThrows; +import org.junit.jupiter.api.io.TempDir; + +/** + * Runs the {@link IcebergTableReadWriteTest} suite against external Iceberg tables: each table is + * created with an explicit {@code LOCATION} under a local temp directory, so Unity Catalog's + * Iceberg REST catalog registers it as EXTERNAL. Mirrors {@code DeltaExternalTableReadWriteTest}. + * + *

Only the local {@code file://} scheme is exercised (the shared base uses a file warehouse); + * the cloud schemes the UC-connector external tests cover would need an S3-protocol backend, which + * is out of scope here. + */ +public class IcebergExternalTableReadWriteTest extends IcebergTableReadWriteTest { + + @TempDir protected File dataDir; + + @Override + protected boolean isManagedTable() { + return false; + } + + @SneakyThrows + private String getLocation(TableSetupOptions options) { + String rawCatalogName = options.getCatalogName().replace("`", ""); + String rawTableName = options.getTableName().replace("`", ""); + return new File(new File(dataDir, rawCatalogName), rawTableName).getCanonicalPath(); + } + + @Override + protected String setupTable(TableSetupOptions options) { + sql(options.createExternalTableSql(getLocation(options))); + return options.fullTableName(); + } +} diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergManagedTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergManagedTableReadWriteTest.java new file mode 100644 index 0000000000..9fe4db0b8c --- /dev/null +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergManagedTableReadWriteTest.java @@ -0,0 +1,20 @@ +package io.unitycatalog.spark; + +/** + * Runs the {@link IcebergTableReadWriteTest} suite against managed Iceberg tables: the table has no + * client-supplied location, so Unity Catalog's Iceberg REST catalog assigns one under the managed + * storage root. Mirrors {@code DeltaManagedTableReadWriteTest}. + */ +public class IcebergManagedTableReadWriteTest extends IcebergTableReadWriteTest { + + @Override + protected boolean isManagedTable() { + return true; + } + + @Override + protected String setupTable(TableSetupOptions options) { + sql(options.createManagedTableSql()); + return options.fullTableName(); + } +} diff --git a/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java new file mode 100644 index 0000000000..7970cbc6c0 --- /dev/null +++ b/connectors/spark/src/test/scala-shims/spark-4.0-4.1/io/unitycatalog/spark/IcebergTableReadWriteTest.java @@ -0,0 +1,79 @@ +package io.unitycatalog.spark; + +import static io.unitycatalog.server.utils.TestUtils.CATALOG_NAME; + +import io.unitycatalog.server.utils.ServerProperties; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.params.provider.Arguments; + +/** + * Shared base for the Spark + Iceberg end-to-end suites. It drives Iceberg tables through a real + * Spark + Iceberg runtime against Unity Catalog's Iceberg REST catalog ({@code + * /api/2.1/unity-catalog/iceberg}), rather than through the UC Spark connector (which has no + * Iceberg support). Spark is configured with Iceberg's own {@code SparkCatalog} (REST), wired by + * {@code BaseSparkIntegrationTest} for every catalog its {@code isIcebergCatalog} marks; the {@code + * /v1/config} handshake returns a {@code prefix} so the standard Iceberg client targets UC's + * catalog-scoped paths, exercising the server's config / namespace / create / commit / load / drop + * endpoints exactly as an external Iceberg client would. + * + *

Storage is a local {@code file://} warehouse: the server writes the first metadata file + * through its native local FileIO and Spark's Iceberg {@code HadoopFileIO} reads and writes data + * files and later metadata to the same directory, so the full create / write / read / commit path + * round-trips without cloud credentials. Cloud credential-vending is covered separately by the + * server-side Iceberg REST catalog tests. + * + *

Concrete subclasses pick managed ({@link IcebergManagedTableReadWriteTest}) or external + * ({@link IcebergExternalTableReadWriteTest}) tables, mirroring the Delta {@code + * DeltaManagedTableReadWriteTest} / {@code DeltaExternalTableReadWriteTest} split. The class reuses + * {@link BaseTableReadWriteTest}'s create/read/write matrix and helpers; the {@code + * ExternalTableReadWriteTest} base is UC-connector-specific, so it is not reused here. + * + *

Lives under {@code src/test/scala-shims/spark-4.0-4.1} so it is compiled only for the Spark + * versions that have an Iceberg Spark runtime ({@code supportIceberg} in the cross-Spark build). + */ +public abstract class IcebergTableReadWriteTest extends BaseTableReadWriteTest { + + @Override + protected boolean isIcebergCatalog(String catalog) { + return true; + } + + @Override + protected void setUpProperties() { + super.setUpProperties(); + // Advertise and accept the Iceberg write endpoints (createTable / updateTable / dropTable); + // otherwise the REST client refuses them as unsupported and the server rejects them. + serverProperties.setProperty(ServerProperties.Property.ICEBERG_TABLE_ENABLED.getKey(), "true"); + } + + @Override + protected String tableFormat() { + return "ICEBERG"; + } + + // These list only the named catalog; the Iceberg suites leave spark_catalog as Spark's built-in + // session catalog (the UC-connector tests also drive spark_catalog). + @Override + protected List supportedCatalogNames() { + return List.of(CATALOG_NAME); + } + + @Override + protected List sessionCatalogNames() { + return List.of(CATALOG_NAME); + } + + // Iceberg supports partitioned creates and CTAS, so every create in the base matrix is expected + // to succeed (the base defaults non-Delta CTAS / non-plain-CREATE to an expected failure). + @Override + protected List expectedCreateFailureMessages(TableSetupOptions options) { + return null; + } + + // The Iceberg REST catalog vends no cloud credentials and these suites use a local warehouse, so + // testTableOperations (and the other cloud-parameterized base tests) run once on the file scheme. + protected static Stream cloudParameters() { + return Stream.of(Arguments.of("file", false, false)); + } +} diff --git a/project/CrossSparkVersions.scala b/project/CrossSparkVersions.scala index 790d076403..705f4d6d6f 100644 --- a/project/CrossSparkVersions.scala +++ b/project/CrossSparkVersions.scala @@ -28,6 +28,10 @@ case class SparkVersionSpec( // identical shims are written once instead of copied per version. sbt ignores non-existent // dirs, so listing a dir a module doesn't use is harmless. additionalSourceDirs: Seq[String], + // Whether an iceberg-spark-runtime is published for this Spark version. Declared per version + // rather than inferred from the version number: Iceberg's Spark runtime lags new Spark releases, + // so a version-math guess would silently break when a runtime is (or isn't) published. + supportIceberg: Boolean = false, requiresSparkCommit: Boolean = false, sourceBuildArtifactBaseVersion: Option[String] = None, sourceBuildDefaultRef: Option[String] = None @@ -67,11 +71,13 @@ object SparkVersionSpec { case l: List[_] => l.map(_.asInstanceOf[String]) } SparkVersionSpec( - v("version").asInstanceOf[String], - sourceDirs, - v.get("requiresSparkCommit").exists(_.asInstanceOf[Boolean]), - v.get("sourceBuildArtifactBaseVersion").map(_.asInstanceOf[String]), - v.get("sourceBuildDefaultRef").map(_.asInstanceOf[String]) + fullVersion = v("version").asInstanceOf[String], + additionalSourceDirs = sourceDirs, + supportIceberg = v.get("supportIceberg").exists(_.asInstanceOf[Boolean]), + requiresSparkCommit = v.get("requiresSparkCommit").exists(_.asInstanceOf[Boolean]), + sourceBuildArtifactBaseVersion = + v.get("sourceBuildArtifactBaseVersion").map(_.asInstanceOf[String]), + sourceBuildDefaultRef = v.get("sourceBuildDefaultRef").map(_.asInstanceOf[String]) ) } (default, versions) @@ -172,6 +178,37 @@ object CrossSparkVersions extends AutoPlugin { else s"${baseName}_${getSparkVersionSpec().shortVersion}" } + /** + * Test-only `iceberg-spark-runtime` for the Spark + Iceberg suites; empty when `supportIceberg` is + * false. Those suites live in a shim dir, so a version with `supportIceberg` must also list that + * dir in its `sourceDirs`, or the dep is added but no Iceberg test is compiled. + * + * The runtime does not bundle the AWS SDK; the fake-S3 Iceberg tests get it from the + * server-shaded test jar (the server depends on it), which precedes this runtime on the test + * classpath. + */ + def icebergSparkTestDeps(icebergVersion: String): Seq[ModuleID] = { + val spec = getSparkVersionSpec() + if (!spec.supportIceberg) Seq.empty + else + Seq( + "org.apache.iceberg" % + s"iceberg-spark-runtime-${spec.shortVersion}_2.13" % icebergVersion % Test) + } + + /** + * Test-only `delta-spark` matching the active Spark version. `delta-spark` is published for every + * supported Spark version, so this is unconditional except for one escape hatch: when UC is + * published to local Maven before Delta is built (the CI pre-Delta publishM2 step), the matching + * Delta artifact may not exist yet, so `-DskipDeltaSpark=true` drops it to avoid a resolution + * failure. + */ + def deltaSparkTestDeps(deltaVersion: String): Seq[ModuleID] = { + if (sys.props.getOrElse("skipDeltaSpark", "false").toBoolean) Seq.empty + else + Seq("io.delta" %% s"delta-spark_${getSparkVersionSpec().shortVersion}" % deltaVersion % Test) + } + /** * Settings for Spark-dependent modules. Includes: * - sparkVersion setting (queryable via `show spark/sparkVersion`) diff --git a/project/spark-versions.json b/project/spark-versions.json index ca5376cc96..0f5612e178 100644 --- a/project/spark-versions.json +++ b/project/spark-versions.json @@ -1,8 +1,8 @@ { "default": "4.1.0", "versions": [ - {"version": "4.0.0", "sourceDirs": ["scala-shims/spark-4.0-4.1"]}, - {"version": "4.1.0", "sourceDirs": ["scala-shims/spark-4.0-4.1"]}, - {"version": "4.2.0", "sourceDirs": ["scala-shims/spark-4.2"]} + {"version": "4.0.0", "sourceDirs": ["scala-shims/spark-4.0-4.1"], "supportIceberg": true}, + {"version": "4.1.0", "sourceDirs": ["scala-shims/spark-4.0-4.1"], "supportIceberg": true}, + {"version": "4.2.0", "sourceDirs": ["scala-shims/spark-4.2"], "supportIceberg": false} ] }