Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 5 additions & 10 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,6 @@ lazy val deltaVersion = sys.props.getOrElse("deltaVersion", "4.3.1")
// This String val is used for libraryDependencies coordinates; the SettingKey is
// queryable in SBT via `show spark/sparkVersion`.
lazy val sparkVersion = CrossSparkVersions.getSparkArtifactVersion()
lazy val sparkMajorMinorVersion = CrossSparkVersions.getSparkVersionSpec().shortVersion

// delta-spark is only needed for tests. When UC is published to local Maven before
// Delta is built (e.g. CI pre-Delta publishM2 step), the matching Delta artifact may
// not exist yet. Pass -DskipDeltaSpark=true to exclude it and avoid resolution failures.
def deltaSparkTestDeps: Seq[ModuleID] =
if (sys.props.getOrElse("skipDeltaSpark", "false").toBoolean) Seq.empty
else Seq("io.delta" %% s"delta-spark_$sparkMajorMinorVersion" % deltaVersion % Test)

// Apache Snapshots resolver is in build/sbt-config/repositories (global).
// No per-module sparkResolvers needed.
Expand Down Expand Up @@ -695,6 +687,8 @@ lazy val spark = (project in file("connectors/spark"))
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
"--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
),
// Arrow-backed vectorized reads need java.nio opened for off-heap memory access.
Test / javaOptions += "--add-opens=java.base/java.nio=ALL-UNNAMED",
javafmtCheckSettings(),
javaCheckstyleSettings("dev/checkstyle-config.xml"),
Compile / compile / javacOptions ++= javacRelease11,
Expand Down Expand Up @@ -743,7 +737,8 @@ lazy val spark = (project in file("connectors/spark"))
"org.apache.hadoop" % "hadoop-aws" % hadoopVersion % Test,
"org.projectlombok" % "lombok" % "1.18.32" % Test,
"com.google.cloud.bigdataoss" % "gcs-connector" % "3.0.2" % Test classifier "shaded",
) ++ deltaSparkTestDeps,
) ++ CrossSparkVersions.deltaSparkTestDeps(deltaVersion)
++ CrossSparkVersions.icebergSparkTestDeps(icebergVersion),
dependencyOverrides ++= Seq(
"com.fasterxml.jackson.core" % "jackson-databind" % sparkJacksonVersion,
"com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkJacksonVersion,
Expand Down Expand Up @@ -833,7 +828,7 @@ lazy val integrationTests = (project in file("integration-tests"))
"org.apache.hadoop" % "hadoop-aws" % hadoopVersion % Test,
"org.apache.hadoop" % "hadoop-azure" % hadoopVersion % Test,
"com.google.cloud.bigdataoss" % "gcs-connector" % "3.0.2" % Test classifier "shaded",
) ++ deltaSparkTestDeps,
) ++ CrossSparkVersions.deltaSparkTestDeps(deltaVersion),
dependencyOverrides ++= Seq(
"com.fasterxml.jackson.core" % "jackson-databind" % sparkJacksonVersion,
"com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkJacksonVersion,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,27 @@ public abstract class BaseSparkIntegrationTest extends BaseCRUDTest {
protected ArrayList<String> createdCatalogs = new ArrayList<>();
protected static final String SPARK_CATALOG = "spark_catalog";

/** The server's Iceberg REST catalog path, served under the UC base path. */
private static final String ICEBERG_REST_PATH = "/api/2.1/unity-catalog/iceberg";

/** S3 bucket with no credentials configured on server - for testing SSP fallback. */
public static final String NO_CREDS_BUCKET = "test-bucket-2-no-creds";

private SchemaOperations schemaOperations;
// Each test would create this session. It will be closed automatically.
protected SparkSession session;

/**
* True when this suite drives the given catalog as Iceberg's own REST {@code SparkCatalog}
* against the server's Iceberg REST catalog, rather than through the UC Spark connector (which
* has no Iceberg support). Per catalog, so one session can mix both (e.g. a Delta join against
* Iceberg); subclasses flip this, and {@link #createSparkSessionWithCatalogs} is the only
* consumer, so no subclass needs to re-implement session creation.
*/
protected boolean isIcebergCatalog(String catalog) {
return false;
}

private void createCommonResources() throws ApiException {
// Common setup operations such as creating a catalog and schema
catalogOperations.createCatalog(
Expand All @@ -52,24 +66,45 @@ protected SparkSession createSparkSessionWithCatalogs(String... catalogs) {
return createSparkSessionWithCatalogs(true, true, catalogs);
}

/** The base Spark builder shared by all integration tests (local master, small shuffle width). */
private SparkSession.Builder newSparkSessionBuilder() {
return SparkSession.builder()
.appName("test")
.master("local[*]")
.config("spark.sql.shuffle.partitions", "4");
}

protected SparkSession createSparkSessionWithCatalogs(
boolean renewCred, boolean credScopedFsEnabled, String... catalogs) {
// renewCred / credScopedFsEnabled are UC-connector options with no Iceberg REST analog.
// A catalog is wired either as an Iceberg REST catalog or through the UC connector
// (UCSingleCatalog), which serves Delta, Parquet, and the other non-Iceberg formats.
boolean anyIceberg = false;
boolean anyUcConnector = false;
for (String catalog : catalogs) {
if (isIcebergCatalog(catalog)) {
anyIceberg = true;
} else {
anyUcConnector = true;
}
}
// Both extensions can coexist in one session, which is what lets a single query join a Delta
// table against an Iceberg table. The UC connector carries Delta support, so its path loads the
// Delta extension (harmless for the non-Delta formats it also serves, e.g. Parquet).
List<String> extensions = new ArrayList<>();
if (anyUcConnector) {
extensions.add("io.delta.sql.DeltaSparkSessionExtension");
}
if (anyIceberg) {
extensions.add("org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions");
}
SparkSession.Builder builder =
SparkSession.builder()
.appName("test")
.master("local[*]")
.config("spark.sql.shuffle.partitions", "4")
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension");
newSparkSessionBuilder().config("spark.sql.extensions", String.join(",", extensions));
for (String catalog : catalogs) {
String catalogConf = "spark.sql.catalog." + catalog;
builder =
builder
.config(catalogConf, UCSingleCatalog.class.getName())
.config(catalogConf + "." + OptionsUtil.URI, serverConfig.getServerUrl())
.config(catalogConf + "." + OptionsUtil.TOKEN, serverConfig.getAuthToken())
.config(catalogConf + "." + OptionsUtil.WAREHOUSE, catalog)
.config(catalogConf + "." + OptionsUtil.RENEW_CREDENTIAL_ENABLED, renewCred)
.config(catalogConf + "." + OptionsUtil.CRED_SCOPED_FS_ENABLED, credScopedFsEnabled);
isIcebergCatalog(catalog)
? icebergCatalogConfig(builder, catalog)
: ucSingleCatalogConfig(builder, catalog, renewCred, credScopedFsEnabled);
if (!List.of(SPARK_CATALOG, CATALOG_NAME).contains(catalog)) {
createTestCatalog(catalog);
}
Expand All @@ -78,9 +113,12 @@ protected SparkSession createSparkSessionWithCatalogs(
// (DeltaLog.checkRequiredConfigurations); otherwise a Delta table -- even in a named UC catalog
// -- fails with DELTA_CONFIGURE_SPARK_SESSION_WITH_EXTENSION_AND_CATALOG. When the caller did
// not configure spark_catalog above, point it at Delta's own catalog to satisfy that check.
if (!List.of(catalogs).contains(SPARK_CATALOG)) {
// This applies to any UC-connector session (the connector may serve Delta); Iceberg has no such
// requirement and these suites address tables only through the named UC catalog, so a
// pure-Iceberg session needs no spark_catalog override and leaves it as the built-in one.
if (anyUcConnector && !List.of(catalogs).contains(SPARK_CATALOG)) {
builder.config(
"spark.sql.catalog." + SPARK_CATALOG, "org.apache.spark.sql.delta.catalog.DeltaCatalog");
catalogConfKey(SPARK_CATALOG), "org.apache.spark.sql.delta.catalog.DeltaCatalog");
}
// Use fake file system for cloud storage so that we can test credentials.
builder.config("spark.hadoop.fs.s3.impl", S3CredentialTestFileSystem.class.getName());
Expand All @@ -89,6 +127,43 @@ protected SparkSession createSparkSessionWithCatalogs(
return builder.getOrCreate();
}

/** The Spark SQL config key prefix for a catalog, e.g. {@code spark.sql.catalog.<name>}. */
private static String catalogConfKey(String catalog) {
return "spark.sql.catalog." + catalog;
}

/** Wires one catalog through the UC Spark connector ({@link UCSingleCatalog}), the Delta path. */
private SparkSession.Builder ucSingleCatalogConfig(
SparkSession.Builder builder,
String catalog,
boolean renewCred,
boolean credScopedFsEnabled) {
String catalogConf = catalogConfKey(catalog);
return builder
.config(catalogConf, UCSingleCatalog.class.getName())
.config(catalogConf + "." + OptionsUtil.URI, serverConfig.getServerUrl())
.config(catalogConf + "." + OptionsUtil.TOKEN, serverConfig.getAuthToken())
.config(catalogConf + "." + OptionsUtil.WAREHOUSE, catalog)
.config(catalogConf + "." + OptionsUtil.RENEW_CREDENTIAL_ENABLED, renewCred)
.config(catalogConf + "." + OptionsUtil.CRED_SCOPED_FS_ENABLED, credScopedFsEnabled);
}

/**
* Wires one catalog as Iceberg's REST {@code SparkCatalog} against the server's Iceberg REST
* catalog, reading and writing data files through Iceberg's {@code HadoopFileIO} on the local
* warehouse.
*/
private SparkSession.Builder icebergCatalogConfig(SparkSession.Builder builder, String catalog) {
String catalogConf = catalogConfKey(catalog);
return builder
.config(catalogConf, "org.apache.iceberg.spark.SparkCatalog")
.config(catalogConf + ".type", "rest")
.config(catalogConf + ".uri", serverConfig.getServerUrl() + ICEBERG_REST_PATH)
.config(catalogConf + ".warehouse", catalog)
.config(catalogConf + ".io-impl", "org.apache.iceberg.hadoop.HadoopFileIO")
.config(catalogConf + ".token", serverConfig.getAuthToken());
}

protected List<Row> sql(String statement, Object... args) {
return session.sql(String.format(statement, args)).collectAsList();
}
Expand Down
Loading
Loading