From bbbae4e61d45cbc29bafedaf432d49111996701b Mon Sep 17 00:00:00 2001 From: timl Date: Fri, 4 Sep 2026 11:44:37 +0800 Subject: [PATCH 1/2] Use renamed cudf-spark-jni artifacts Update cuDF Spark to consume JNI artifacts under their new project identity. - add cudf-spark-jni artifact ID and version properties - resolve Maven dependencies, shading exclusions, and unpack steps through the artifact ID property - load the renamed cudf-spark-jni version metadata resource - update dependency discovery, nightly controls, Scala 2.13 POMs, documentation, and review configuration Centralizing the artifact ID matches the cudf-spark-private dependency pattern and allows the coordinate to be overridden without editing each module. Signed-off-by: timl --- .github/workflows/mvn-verify-check/get-deps-sha1.sh | 4 ++-- .greptile/config.json | 6 +++--- .greptile/rules.md | 2 +- aggregator/pom.xml | 2 +- api_validation/README.md | 2 +- api_validation/pom.xml | 2 +- build/make-scala-version-build-files.sh | 4 ++-- dist/pom.xml | 2 +- docs/additional-functionality/advanced_configs.md | 2 +- iceberg-common/pom.xml | 2 +- jenkins/spark-nightly-build.sh | 2 +- pom.xml | 9 +++++---- scala2.13/aggregator/pom.xml | 2 +- scala2.13/api_validation/pom.xml | 2 +- scala2.13/dist/pom.xml | 2 +- scala2.13/iceberg-common/pom.xml | 2 +- scala2.13/pom.xml | 9 +++++---- scala2.13/shuffle-plugin/pom.xml | 2 +- scala2.13/sql-plugin-api/pom.xml | 4 ++-- scala2.13/sql-plugin-columnar/pom.xml | 2 +- scala2.13/sql-plugin-fileio/pom.xml | 2 +- scala2.13/sql-plugin/pom.xml | 2 +- scala2.13/tests/pom.xml | 2 +- scala2.13/udf-compiler/pom.xml | 2 +- shuffle-plugin/pom.xml | 2 +- sql-plugin-api/pom.xml | 4 ++-- sql-plugin-columnar/pom.xml | 2 +- sql-plugin-fileio/pom.xml | 2 +- sql-plugin/pom.xml | 2 +- .../src/main/scala/com/nvidia/spark/rapids/Plugin.scala | 4 ++-- .../main/scala/com/nvidia/spark/rapids/RapidsConf.scala | 2 +- tests/pom.xml | 2 +- udf-compiler/pom.xml | 2 +- 33 files changed, 48 insertions(+), 46 deletions(-) diff --git a/.github/workflows/mvn-verify-check/get-deps-sha1.sh b/.github/workflows/mvn-verify-check/get-deps-sha1.sh index 79c07ac905d..9d078d78bbd 100755 --- a/.github/workflows/mvn-verify-check/get-deps-sha1.sh +++ b/.github/workflows/mvn-verify-check/get-deps-sha1.sh @@ -18,10 +18,10 @@ set -e scala_ver=${1:-"2.12"} base_URL="https://central.sonatype.com/repository/maven-snapshots/com/nvidia" -project_jni="spark-rapids-jni" +project_jni="cudf-spark-jni" project_private="cudf-spark-private_${scala_ver}" -jni_ver=$(mvn help:evaluate -q -pl dist -Dexpression=spark-rapids-jni.version -DforceStdout) +jni_ver=$(mvn help:evaluate -q -pl dist -Dexpression=cudf-spark-jni.version -DforceStdout) private_ver=$(mvn help:evaluate -q -pl dist -Dexpression=cudf-spark-private.version -DforceStdout) get_latest_snapshot_version() { diff --git a/.greptile/config.json b/.greptile/config.json index c785a64e989..550712a3880 100644 --- a/.greptile/config.json +++ b/.greptile/config.json @@ -6,11 +6,11 @@ "patternRepositories": [ "rapidsai/cudf", - "NVIDIA/spark-rapids-jni", + "NVIDIA/cudf-spark-jni", "apache/spark" ], - "instructions": "Spark plugin for GPU-accelerated data processing using RAPIDS cuDF. Maven-based, Scala/Java with Python integration tests. Key modules: sql-plugin/, shuffle-plugin/, iceberg/, delta-lake/, tests/, integration_tests/. Version-specific shims at sql-plugin/src/main/spark{VERSION}/. Review rules are in .greptile/rules.md. Key files are listed in .greptile/files.json. Also reference AGENTS.md for build commands, project structure, and coding conventions. IMPORTANT: This project depends on cuDF (rapidsai/cudf) for GPU compute kernels and spark-rapids-jni (NVIDIA/spark-rapids-jni) for JNI bindings. When reviewing code that calls cuDF/JNI APIs, cross-reference the upstream repos to verify API usage correctness, especially for: column operations, memory allocation semantics, stream handling, and error codes. This project also shims against multiple Apache Spark versions (apache/spark). When reviewing shim code or GPU operator implementations, cross-reference apache/spark to verify that the GPU behavior matches the corresponding CPU implementation in the target Spark version, especially for: expression evaluation semantics, null handling, type coercion, and catalog/partition behavior.", + "instructions": "Spark plugin for GPU-accelerated data processing using RAPIDS cuDF. Maven-based, Scala/Java with Python integration tests. Key modules: sql-plugin/, shuffle-plugin/, iceberg/, delta-lake/, tests/, integration_tests/. Version-specific shims at sql-plugin/src/main/spark{VERSION}/. Review rules are in .greptile/rules.md. Key files are listed in .greptile/files.json. Also reference AGENTS.md for build commands, project structure, and coding conventions. IMPORTANT: This project depends on cuDF (rapidsai/cudf) for GPU compute kernels and cudf-spark-jni (NVIDIA/cudf-spark-jni) for JNI bindings. When reviewing code that calls cuDF/JNI APIs, cross-reference the upstream repos to verify API usage correctness, especially for: column operations, memory allocation semantics, stream handling, and error codes. This project also shims against multiple Apache Spark versions (apache/spark). When reviewing shim code or GPU operator implementations, cross-reference apache/spark to verify that the GPU behavior matches the corresponding CPU implementation in the target Spark version, especially for: expression evaluation semantics, null handling, type coercion, and catalog/partition behavior.", "rules": [ { @@ -117,7 +117,7 @@ }, { "id": "upstream-dependency-boundary", - "rule": "Code touching JNI calls (cudf/spark-rapids-jni) or private API interfaces (spark-rapids-private) may break from concurrent upstream SNAPSHOT changes. Review for assumptions about upstream behavior. Cross-reference rapidsai/cudf for correct function signatures, null handling semantics, and memory ownership rules at JNI boundaries.", + "rule": "Code touching JNI calls (cudf/cudf-spark-jni) or private API interfaces (spark-rapids-private) may break from concurrent upstream SNAPSHOT changes. Review for assumptions about upstream behavior. Cross-reference rapidsai/cudf for correct function signatures, null handling semantics, and memory ownership rules at JNI boundaries.", "scope": ["sql-plugin/src/**", "shuffle-plugin/src/**"], "severity": "high" }, diff --git a/.greptile/rules.md b/.greptile/rules.md index 00e0b28a873..e0c2e6dcfee 100644 --- a/.greptile/rules.md +++ b/.greptile/rules.md @@ -17,6 +17,6 @@ shim layer architecture, see `AGENTS.md` at the repo root. - [ ] H5: Configuration — new RapidsConf without docs/defaults; should use .internal() if not user-visible; new features default off - [ ] H6: Magic numbers — unexplained numeric literals without named constants or comments - [ ] H7: Pre-merge CI gaps — only selected shims run unit tests; feature-gated tests need explicit enable; limited Scala 2.13 coverage -- [ ] H8: Upstream dependencies — SNAPSHOT changes from spark-rapids-jni/cudf may break; verify API usage against upstream repos +- [ ] H8: Upstream dependencies — SNAPSHOT changes from cudf-spark-jni/cudf may break; verify API usage against upstream repos - [ ] H9: Databricks coverage — change may behave differently on Databricks (filesystem/path semantics, plan-string/optimizer assertions, DBR Spark-fork runtime) but the PR title lacks [databricks] and the diff doesn't touch a *db* shim path or a path containing `databricks` (either auto-triggers DB CI). Recommend [databricks] or confirm DB validation (see AGENTS.md § PR title tags) - [ ] H10: Performance checklist — report `Performance: Not required` as a high-severity finding unless the PR is documentation-only or test-only, or its description gives a verifiable reason the change cannot affect runtime performance. A bug-fix label, small diff, or rarely used path is not by itself an exemption. When uncertain, flag diff --git a/aggregator/pom.xml b/aggregator/pom.xml index 90c255f558a..a85d658131d 100644 --- a/aggregator/pom.xml +++ b/aggregator/pom.xml @@ -167,7 +167,7 @@ org.slf4j:* - com.nvidia:spark-rapids-jni:* + com.nvidia:${cudf-spark-jni.artifactId}:* org.openucx:jucx:* diff --git a/api_validation/README.md b/api_validation/README.md index 482b3e76a58..9aa80c65880 100644 --- a/api_validation/README.md +++ b/api_validation/README.md @@ -11,7 +11,7 @@ Validation fails when: # Dependencies -It requires spark-rapids-jni, rapids-4-spark and Spark jars. +It requires cudf-spark-jni, rapids-4-spark and Spark jars. # Running the script diff --git a/api_validation/pom.xml b/api_validation/pom.xml index 9bbbd327b6f..514476558a8 100644 --- a/api_validation/pom.xml +++ b/api_validation/pom.xml @@ -43,7 +43,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} provided diff --git a/build/make-scala-version-build-files.sh b/build/make-scala-version-build-files.sh index 43891129177..01c36e79d71 100755 --- a/build/make-scala-version-build-files.sh +++ b/build/make-scala-version-build-files.sh @@ -105,7 +105,7 @@ sed_i '//,/\${spark[0-9]\+\.version} in parent POM # Match any scala binary version to ensure idempotency -sed_i '//,/[0-9]*\.[0-9]*[0-9]*\.[0-9]*'$TO_VERSION'/,/[0-9]*\.[0-9]*[0-9]*\.[0-9]*'$TO_VERSION'/,/[0-9]*\.[0-9]*\.[0-9]*[0-9]*\.[0-9]*\.[0-9]*'$SCALA_VERSION'/,/[0-9]*\.[0-9]*\.[0-9]*[0-9]*\.[0-9]*\.[0-9]*'$SCALA_VERSION' com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} META-INF/** ${project.build.directory}/jni-deps diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 5b5b60be78a..d337c6843f2 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -71,7 +71,7 @@ Name | Description | Default Value | Applicable at spark.rapids.shuffle.ucx.managementServerHost|The host to be used to start the management server|null|Startup spark.rapids.shuffle.ucx.useWakeup|When set to true, use UCX's event-based progress (epoll) in order to wake up the progress thread when needed, instead of a hot loop.|true|Startup spark.rapids.sql.acceleratedColumnarToRow.enabled|When set to true (default) the GPU columnar-to-row transition uses the GPU transpose kernel (AcceleratedColumnarToRowIterator) for wide fixed-width / STRING schemas. Setting it to false forces the slower per-row CPU iterator (ColumnarToRowIterator). Mainly useful for troubleshooting and performance comparisons; production workloads should leave this on.|true|Runtime -spark.rapids.sql.allowMultipleJars|Allow multiple rapids-4-spark, spark-rapids-jni, and cudf jars on the classpath. Spark will take the first one it finds, so the version may not be expected. Possisble values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same revision, NEVER: do not allow multiple jars at all.|SAME_REVISION|Startup +spark.rapids.sql.allowMultipleJars|Allow multiple rapids-4-spark, cudf-spark-jni, and cudf jars on the classpath. Spark will take the first one it finds, so the version may not be expected. Possisble values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same revision, NEVER: do not allow multiple jars at all.|SAME_REVISION|Startup spark.rapids.sql.castDecimalToFloat.enabled|Casting from decimal to floating point types on the GPU returns results that have tiny difference compared to results returned from CPU.|true|Runtime spark.rapids.sql.castFloatToDecimal.enabled|Casting from floating point types to decimal on the GPU returns results that have tiny difference compared to results returned from CPU.|true|Runtime spark.rapids.sql.castFloatToIntegralTypes.enabled|Casting from floating point types to integral types on the GPU supports a slightly different range of values when using Spark 3.1.0 or later. Refer to the CAST documentation for more details.|true|Runtime diff --git a/iceberg-common/pom.xml b/iceberg-common/pom.xml index 57520e19d74..5d0624758bf 100644 --- a/iceberg-common/pom.xml +++ b/iceberg-common/pom.xml @@ -42,7 +42,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index 874a9e74b34..c5df79cd5e9 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -17,7 +17,7 @@ set -ex -## MVN_OPT : maven options environment, e.g. MVN_OPT='-Dspark-rapids-jni.version=xxx' to specify spark-rapids-jni dependency's version. +## MVN_OPT : maven options environment, e.g. MVN_OPT='-Dcudf-spark-jni.version=xxx' to specify cudf-spark-jni dependency's version. MVN_SETTINGS=${MVN_SETTINGS:-"jenkins/settings.xml"} export MVN="mvn -s $MVN_SETTINGS -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 ${MVN_OPT} -Psource-javadoc" diff --git a/pom.xml b/pom.xml index 4b776c3a0a1..746fbf8ab3b 100644 --- a/pom.xml +++ b/pom.xml @@ -1061,7 +1061,8 @@ spark${buildver} cuda12 ${cuda.version} - 26.10.0-SNAPSHOT + 26.10.0-SNAPSHOT + cudf-spark-jni 26.10.0-SNAPSHOT cudf-spark-private_${scala.binary.version} 2.12 @@ -1275,8 +1276,8 @@ com.nvidia - spark-rapids-jni - ${spark-rapids-jni.version} + ${cudf-spark-jni.artifactId} + ${cudf-spark-jni.version} ${jni.classifier} @@ -1572,7 +1573,7 @@ This will force full Scala code rebuild in downstream modules. failonerror="false"> - + diff --git a/scala2.13/aggregator/pom.xml b/scala2.13/aggregator/pom.xml index ced3cd10363..8069e4e5871 100644 --- a/scala2.13/aggregator/pom.xml +++ b/scala2.13/aggregator/pom.xml @@ -167,7 +167,7 @@ org.slf4j:* - com.nvidia:spark-rapids-jni:* + com.nvidia:${cudf-spark-jni.artifactId}:* org.openucx:jucx:* diff --git a/scala2.13/api_validation/pom.xml b/scala2.13/api_validation/pom.xml index 18dd39f2b1b..72c5e50eb6a 100644 --- a/scala2.13/api_validation/pom.xml +++ b/scala2.13/api_validation/pom.xml @@ -43,7 +43,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} provided diff --git a/scala2.13/dist/pom.xml b/scala2.13/dist/pom.xml index c9fe0648384..cc8546c735d 100644 --- a/scala2.13/dist/pom.xml +++ b/scala2.13/dist/pom.xml @@ -445,7 +445,7 @@ self.log("... OK") com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} META-INF/** ${project.build.directory}/jni-deps diff --git a/scala2.13/iceberg-common/pom.xml b/scala2.13/iceberg-common/pom.xml index 0390c3fc5d7..a972444c716 100644 --- a/scala2.13/iceberg-common/pom.xml +++ b/scala2.13/iceberg-common/pom.xml @@ -42,7 +42,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index bd11ffcbcb0..a59788cc7ef 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -1061,7 +1061,8 @@ spark${buildver} cuda12 ${cuda.version} - 26.10.0-SNAPSHOT + 26.10.0-SNAPSHOT + cudf-spark-jni 26.10.0-SNAPSHOT cudf-spark-private_${scala.binary.version} 2.13 @@ -1275,8 +1276,8 @@ com.nvidia - spark-rapids-jni - ${spark-rapids-jni.version} + ${cudf-spark-jni.artifactId} + ${cudf-spark-jni.version} ${jni.classifier} @@ -1572,7 +1573,7 @@ This will force full Scala code rebuild in downstream modules. failonerror="false"> - + diff --git a/scala2.13/shuffle-plugin/pom.xml b/scala2.13/shuffle-plugin/pom.xml index 3aac09f8218..dcaddc2ca6f 100644 --- a/scala2.13/shuffle-plugin/pom.xml +++ b/scala2.13/shuffle-plugin/pom.xml @@ -39,7 +39,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/sql-plugin-api/pom.xml b/scala2.13/sql-plugin-api/pom.xml index 96448cb7f2f..40310173036 100644 --- a/scala2.13/sql-plugin-api/pom.xml +++ b/scala2.13/sql-plugin-api/pom.xml @@ -38,7 +38,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} @@ -132,4 +132,4 @@ - \ No newline at end of file + diff --git a/scala2.13/sql-plugin-columnar/pom.xml b/scala2.13/sql-plugin-columnar/pom.xml index 65b2b08015e..631f05f66dd 100644 --- a/scala2.13/sql-plugin-columnar/pom.xml +++ b/scala2.13/sql-plugin-columnar/pom.xml @@ -41,7 +41,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/sql-plugin-fileio/pom.xml b/scala2.13/sql-plugin-fileio/pom.xml index 11f72e491ab..de096ad80a4 100644 --- a/scala2.13/sql-plugin-fileio/pom.xml +++ b/scala2.13/sql-plugin-fileio/pom.xml @@ -41,7 +41,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/sql-plugin/pom.xml b/scala2.13/sql-plugin/pom.xml index 07b24738358..1fc2d30d773 100644 --- a/scala2.13/sql-plugin/pom.xml +++ b/scala2.13/sql-plugin/pom.xml @@ -45,7 +45,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/tests/pom.xml b/scala2.13/tests/pom.xml index 92e28b94788..e19d25c80dd 100644 --- a/scala2.13/tests/pom.xml +++ b/scala2.13/tests/pom.xml @@ -69,7 +69,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/udf-compiler/pom.xml b/scala2.13/udf-compiler/pom.xml index 73a3fad9b9b..1b96ece0598 100644 --- a/scala2.13/udf-compiler/pom.xml +++ b/scala2.13/udf-compiler/pom.xml @@ -39,7 +39,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/shuffle-plugin/pom.xml b/shuffle-plugin/pom.xml index 5e4b9051ddc..eef2e93f9cb 100644 --- a/shuffle-plugin/pom.xml +++ b/shuffle-plugin/pom.xml @@ -39,7 +39,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/sql-plugin-api/pom.xml b/sql-plugin-api/pom.xml index 04d1ec1dca5..dabce7f1b03 100644 --- a/sql-plugin-api/pom.xml +++ b/sql-plugin-api/pom.xml @@ -38,7 +38,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} @@ -132,4 +132,4 @@ - \ No newline at end of file + diff --git a/sql-plugin-columnar/pom.xml b/sql-plugin-columnar/pom.xml index 3ab08e912eb..3f2d8666a11 100644 --- a/sql-plugin-columnar/pom.xml +++ b/sql-plugin-columnar/pom.xml @@ -41,7 +41,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/sql-plugin-fileio/pom.xml b/sql-plugin-fileio/pom.xml index 01f9c7e3600..40028e94837 100644 --- a/sql-plugin-fileio/pom.xml +++ b/sql-plugin-fileio/pom.xml @@ -41,7 +41,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml index deb838686ee..7fc717b9bb5 100644 --- a/sql-plugin/pom.xml +++ b/sql-plugin/pom.xml @@ -45,7 +45,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala index ee616a43b28..658edd804cd 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala @@ -82,7 +82,7 @@ case class ColumnarOverrideRules(sparkSession: SparkSession) extends ColumnarRul object RapidsPluginUtils extends Logging { val CUDF_PROPS_FILENAME = "cudf-java-version-info.properties" - val JNI_PROPS_FILENAME = "spark-rapids-jni-version-info.properties" + val JNI_PROPS_FILENAME = "cudf-spark-jni-version-info.properties" val PLUGIN_PROPS_FILENAME = "rapids4spark-version-info.properties" private val PRIVATE_PROPS_FILENAME = "cudf-spark-private-version-info.properties" @@ -210,7 +210,7 @@ object RapidsPluginUtils extends Logging { def detectMultipleJars(conf: RapidsConf): Unit = { detectMultipleJar(PLUGIN_PROPS_FILENAME, "rapids-4-spark", conf) - detectMultipleJar(JNI_PROPS_FILENAME, "spark-rapids-jni", conf) + detectMultipleJar(JNI_PROPS_FILENAME, "cudf-spark-jni", conf) detectMultipleJar(CUDF_PROPS_FILENAME, "cudf", conf) } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index ff2533e7d4c..d6059c8f886 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -2569,7 +2569,7 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. val ALLOW_MULTIPLE_JARS = conf("spark.rapids.sql.allowMultipleJars") .startupOnly() - .doc("Allow multiple rapids-4-spark, spark-rapids-jni, and cudf jars on the classpath. " + + .doc("Allow multiple rapids-4-spark, cudf-spark-jni, and cudf jars on the classpath. " + "Spark will take the first one it finds, so the version may not be expected. Possisble " + "values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same " + "revision, NEVER: do not allow multiple jars at all.") diff --git a/tests/pom.xml b/tests/pom.xml index d26afb68404..4c6e04a60e1 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -69,7 +69,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/udf-compiler/pom.xml b/udf-compiler/pom.xml index 5e64c60363e..fa891354220 100644 --- a/udf-compiler/pom.xml +++ b/udf-compiler/pom.xml @@ -39,7 +39,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} From 125357e00da73bc48379df900f1d3d44a7ffc69b Mon Sep 17 00:00:00 2001 From: timl Date: Wed, 9 Sep 2026 16:35:35 +0800 Subject: [PATCH 2/2] Address review feedback for JNI artifact rename Resolve the cudf-spark-jni artifact ID from Maven in the dependency hash lookup so command-line property overrides are honored. Update stale spark-rapids-jni and spark-rapids-private references in the aggregator shading comments and Greptile guidance. Keep the generated Scala 2.13 aggregator POM synchronized. Correct the allowMultipleJars configuration description typo in RapidsConf and the generated advanced configuration documentation. Signed-off-by: timl --- .github/workflows/mvn-verify-check/get-deps-sha1.sh | 2 +- .greptile/config.json | 2 +- aggregator/pom.xml | 2 +- docs/additional-functionality/advanced_configs.md | 2 +- scala2.13/aggregator/pom.xml | 2 +- .../src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mvn-verify-check/get-deps-sha1.sh b/.github/workflows/mvn-verify-check/get-deps-sha1.sh index 9d078d78bbd..8cc34730b42 100755 --- a/.github/workflows/mvn-verify-check/get-deps-sha1.sh +++ b/.github/workflows/mvn-verify-check/get-deps-sha1.sh @@ -18,7 +18,7 @@ set -e scala_ver=${1:-"2.12"} base_URL="https://central.sonatype.com/repository/maven-snapshots/com/nvidia" -project_jni="cudf-spark-jni" +project_jni=$(mvn help:evaluate -q -pl dist -Dexpression=cudf-spark-jni.artifactId -DforceStdout) project_private="cudf-spark-private_${scala_ver}" jni_ver=$(mvn help:evaluate -q -pl dist -Dexpression=cudf-spark-jni.version -DforceStdout) diff --git a/.greptile/config.json b/.greptile/config.json index 550712a3880..a329a9e4fcd 100644 --- a/.greptile/config.json +++ b/.greptile/config.json @@ -117,7 +117,7 @@ }, { "id": "upstream-dependency-boundary", - "rule": "Code touching JNI calls (cudf/cudf-spark-jni) or private API interfaces (spark-rapids-private) may break from concurrent upstream SNAPSHOT changes. Review for assumptions about upstream behavior. Cross-reference rapidsai/cudf for correct function signatures, null handling semantics, and memory ownership rules at JNI boundaries.", + "rule": "Code touching JNI calls (cudf/cudf-spark-jni) or private API interfaces (cudf-spark-private) may break from concurrent upstream SNAPSHOT changes. Review for assumptions about upstream behavior. Cross-reference rapidsai/cudf for correct function signatures, null handling semantics, and memory ownership rules at JNI boundaries.", "scope": ["sql-plugin/src/**", "shuffle-plugin/src/**"], "severity": "high" }, diff --git a/aggregator/pom.xml b/aggregator/pom.xml index a85d658131d..a63b020ef14 100644 --- a/aggregator/pom.xml +++ b/aggregator/pom.xml @@ -166,7 +166,7 @@ org.slf4j:* - + com.nvidia:${cudf-spark-jni.artifactId}:* org.openucx:jucx:* diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index d337c6843f2..eaa5a72e4d5 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -71,7 +71,7 @@ Name | Description | Default Value | Applicable at spark.rapids.shuffle.ucx.managementServerHost|The host to be used to start the management server|null|Startup spark.rapids.shuffle.ucx.useWakeup|When set to true, use UCX's event-based progress (epoll) in order to wake up the progress thread when needed, instead of a hot loop.|true|Startup spark.rapids.sql.acceleratedColumnarToRow.enabled|When set to true (default) the GPU columnar-to-row transition uses the GPU transpose kernel (AcceleratedColumnarToRowIterator) for wide fixed-width / STRING schemas. Setting it to false forces the slower per-row CPU iterator (ColumnarToRowIterator). Mainly useful for troubleshooting and performance comparisons; production workloads should leave this on.|true|Runtime -spark.rapids.sql.allowMultipleJars|Allow multiple rapids-4-spark, cudf-spark-jni, and cudf jars on the classpath. Spark will take the first one it finds, so the version may not be expected. Possisble values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same revision, NEVER: do not allow multiple jars at all.|SAME_REVISION|Startup +spark.rapids.sql.allowMultipleJars|Allow multiple rapids-4-spark, cudf-spark-jni, and cudf jars on the classpath. Spark will take the first one it finds, so the version may not be expected. Possible values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same revision, NEVER: do not allow multiple jars at all.|SAME_REVISION|Startup spark.rapids.sql.castDecimalToFloat.enabled|Casting from decimal to floating point types on the GPU returns results that have tiny difference compared to results returned from CPU.|true|Runtime spark.rapids.sql.castFloatToDecimal.enabled|Casting from floating point types to decimal on the GPU returns results that have tiny difference compared to results returned from CPU.|true|Runtime spark.rapids.sql.castFloatToIntegralTypes.enabled|Casting from floating point types to integral types on the GPU supports a slightly different range of values when using Spark 3.1.0 or later. Refer to the CAST documentation for more details.|true|Runtime diff --git a/scala2.13/aggregator/pom.xml b/scala2.13/aggregator/pom.xml index 8069e4e5871..746e83ad015 100644 --- a/scala2.13/aggregator/pom.xml +++ b/scala2.13/aggregator/pom.xml @@ -166,7 +166,7 @@ org.slf4j:* - + com.nvidia:${cudf-spark-jni.artifactId}:* org.openucx:jucx:* diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index d6059c8f886..a53f29b2990 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -2570,7 +2570,7 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. val ALLOW_MULTIPLE_JARS = conf("spark.rapids.sql.allowMultipleJars") .startupOnly() .doc("Allow multiple rapids-4-spark, cudf-spark-jni, and cudf jars on the classpath. " + - "Spark will take the first one it finds, so the version may not be expected. Possisble " + + "Spark will take the first one it finds, so the version may not be expected. Possible " + "values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same " + "revision, NEVER: do not allow multiple jars at all.") .stringConf