diff --git a/.github/workflows/mvn-verify-check/get-deps-sha1.sh b/.github/workflows/mvn-verify-check/get-deps-sha1.sh index 79c07ac905d..8cc34730b42 100755 --- a/.github/workflows/mvn-verify-check/get-deps-sha1.sh +++ b/.github/workflows/mvn-verify-check/get-deps-sha1.sh @@ -18,10 +18,10 @@ set -e scala_ver=${1:-"2.12"} base_URL="https://central.sonatype.com/repository/maven-snapshots/com/nvidia" -project_jni="spark-rapids-jni" +project_jni=$(mvn help:evaluate -q -pl dist -Dexpression=cudf-spark-jni.artifactId -DforceStdout) project_private="cudf-spark-private_${scala_ver}" -jni_ver=$(mvn help:evaluate -q -pl dist -Dexpression=spark-rapids-jni.version -DforceStdout) +jni_ver=$(mvn help:evaluate -q -pl dist -Dexpression=cudf-spark-jni.version -DforceStdout) private_ver=$(mvn help:evaluate -q -pl dist -Dexpression=cudf-spark-private.version -DforceStdout) get_latest_snapshot_version() { diff --git a/.greptile/config.json b/.greptile/config.json index c785a64e989..a329a9e4fcd 100644 --- a/.greptile/config.json +++ b/.greptile/config.json @@ -6,11 +6,11 @@ "patternRepositories": [ "rapidsai/cudf", - "NVIDIA/spark-rapids-jni", + "NVIDIA/cudf-spark-jni", "apache/spark" ], - "instructions": "Spark plugin for GPU-accelerated data processing using RAPIDS cuDF. Maven-based, Scala/Java with Python integration tests. Key modules: sql-plugin/, shuffle-plugin/, iceberg/, delta-lake/, tests/, integration_tests/. Version-specific shims at sql-plugin/src/main/spark{VERSION}/. Review rules are in .greptile/rules.md. Key files are listed in .greptile/files.json. Also reference AGENTS.md for build commands, project structure, and coding conventions. IMPORTANT: This project depends on cuDF (rapidsai/cudf) for GPU compute kernels and spark-rapids-jni (NVIDIA/spark-rapids-jni) for JNI bindings. When reviewing code that calls cuDF/JNI APIs, cross-reference the upstream repos to verify API usage correctness, especially for: column operations, memory allocation semantics, stream handling, and error codes. This project also shims against multiple Apache Spark versions (apache/spark). When reviewing shim code or GPU operator implementations, cross-reference apache/spark to verify that the GPU behavior matches the corresponding CPU implementation in the target Spark version, especially for: expression evaluation semantics, null handling, type coercion, and catalog/partition behavior.", + "instructions": "Spark plugin for GPU-accelerated data processing using RAPIDS cuDF. Maven-based, Scala/Java with Python integration tests. Key modules: sql-plugin/, shuffle-plugin/, iceberg/, delta-lake/, tests/, integration_tests/. Version-specific shims at sql-plugin/src/main/spark{VERSION}/. Review rules are in .greptile/rules.md. Key files are listed in .greptile/files.json. Also reference AGENTS.md for build commands, project structure, and coding conventions. IMPORTANT: This project depends on cuDF (rapidsai/cudf) for GPU compute kernels and cudf-spark-jni (NVIDIA/cudf-spark-jni) for JNI bindings. When reviewing code that calls cuDF/JNI APIs, cross-reference the upstream repos to verify API usage correctness, especially for: column operations, memory allocation semantics, stream handling, and error codes. This project also shims against multiple Apache Spark versions (apache/spark). When reviewing shim code or GPU operator implementations, cross-reference apache/spark to verify that the GPU behavior matches the corresponding CPU implementation in the target Spark version, especially for: expression evaluation semantics, null handling, type coercion, and catalog/partition behavior.", "rules": [ { @@ -117,7 +117,7 @@ }, { "id": "upstream-dependency-boundary", - "rule": "Code touching JNI calls (cudf/spark-rapids-jni) or private API interfaces (spark-rapids-private) may break from concurrent upstream SNAPSHOT changes. Review for assumptions about upstream behavior. Cross-reference rapidsai/cudf for correct function signatures, null handling semantics, and memory ownership rules at JNI boundaries.", + "rule": "Code touching JNI calls (cudf/cudf-spark-jni) or private API interfaces (cudf-spark-private) may break from concurrent upstream SNAPSHOT changes. Review for assumptions about upstream behavior. Cross-reference rapidsai/cudf for correct function signatures, null handling semantics, and memory ownership rules at JNI boundaries.", "scope": ["sql-plugin/src/**", "shuffle-plugin/src/**"], "severity": "high" }, diff --git a/.greptile/rules.md b/.greptile/rules.md index 00e0b28a873..e0c2e6dcfee 100644 --- a/.greptile/rules.md +++ b/.greptile/rules.md @@ -17,6 +17,6 @@ shim layer architecture, see `AGENTS.md` at the repo root. - [ ] H5: Configuration — new RapidsConf without docs/defaults; should use .internal() if not user-visible; new features default off - [ ] H6: Magic numbers — unexplained numeric literals without named constants or comments - [ ] H7: Pre-merge CI gaps — only selected shims run unit tests; feature-gated tests need explicit enable; limited Scala 2.13 coverage -- [ ] H8: Upstream dependencies — SNAPSHOT changes from spark-rapids-jni/cudf may break; verify API usage against upstream repos +- [ ] H8: Upstream dependencies — SNAPSHOT changes from cudf-spark-jni/cudf may break; verify API usage against upstream repos - [ ] H9: Databricks coverage — change may behave differently on Databricks (filesystem/path semantics, plan-string/optimizer assertions, DBR Spark-fork runtime) but the PR title lacks [databricks] and the diff doesn't touch a *db* shim path or a path containing `databricks` (either auto-triggers DB CI). Recommend [databricks] or confirm DB validation (see AGENTS.md § PR title tags) - [ ] H10: Performance checklist — report `Performance: Not required` as a high-severity finding unless the PR is documentation-only or test-only, or its description gives a verifiable reason the change cannot affect runtime performance. A bug-fix label, small diff, or rarely used path is not by itself an exemption. When uncertain, flag diff --git a/aggregator/pom.xml b/aggregator/pom.xml index 90c255f558a..a63b020ef14 100644 --- a/aggregator/pom.xml +++ b/aggregator/pom.xml @@ -166,8 +166,8 @@ org.slf4j:* - - com.nvidia:spark-rapids-jni:* + + com.nvidia:${cudf-spark-jni.artifactId}:* org.openucx:jucx:* diff --git a/api_validation/README.md b/api_validation/README.md index 482b3e76a58..9aa80c65880 100644 --- a/api_validation/README.md +++ b/api_validation/README.md @@ -11,7 +11,7 @@ Validation fails when: # Dependencies -It requires spark-rapids-jni, rapids-4-spark and Spark jars. +It requires cudf-spark-jni, rapids-4-spark and Spark jars. # Running the script diff --git a/api_validation/pom.xml b/api_validation/pom.xml index 9bbbd327b6f..514476558a8 100644 --- a/api_validation/pom.xml +++ b/api_validation/pom.xml @@ -43,7 +43,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} provided diff --git a/build/make-scala-version-build-files.sh b/build/make-scala-version-build-files.sh index 43891129177..01c36e79d71 100755 --- a/build/make-scala-version-build-files.sh +++ b/build/make-scala-version-build-files.sh @@ -105,7 +105,7 @@ sed_i '//,/\${spark[0-9]\+\.version} in parent POM # Match any scala binary version to ensure idempotency -sed_i '//,/[0-9]*\.[0-9]*[0-9]*\.[0-9]*'$TO_VERSION'/,/[0-9]*\.[0-9]*[0-9]*\.[0-9]*'$TO_VERSION'/,/[0-9]*\.[0-9]*\.[0-9]*[0-9]*\.[0-9]*\.[0-9]*'$SCALA_VERSION'/,/[0-9]*\.[0-9]*\.[0-9]*[0-9]*\.[0-9]*\.[0-9]*'$SCALA_VERSION' com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} META-INF/** ${project.build.directory}/jni-deps diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 5b5b60be78a..eaa5a72e4d5 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -71,7 +71,7 @@ Name | Description | Default Value | Applicable at spark.rapids.shuffle.ucx.managementServerHost|The host to be used to start the management server|null|Startup spark.rapids.shuffle.ucx.useWakeup|When set to true, use UCX's event-based progress (epoll) in order to wake up the progress thread when needed, instead of a hot loop.|true|Startup spark.rapids.sql.acceleratedColumnarToRow.enabled|When set to true (default) the GPU columnar-to-row transition uses the GPU transpose kernel (AcceleratedColumnarToRowIterator) for wide fixed-width / STRING schemas. Setting it to false forces the slower per-row CPU iterator (ColumnarToRowIterator). Mainly useful for troubleshooting and performance comparisons; production workloads should leave this on.|true|Runtime -spark.rapids.sql.allowMultipleJars|Allow multiple rapids-4-spark, spark-rapids-jni, and cudf jars on the classpath. Spark will take the first one it finds, so the version may not be expected. Possisble values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same revision, NEVER: do not allow multiple jars at all.|SAME_REVISION|Startup +spark.rapids.sql.allowMultipleJars|Allow multiple rapids-4-spark, cudf-spark-jni, and cudf jars on the classpath. Spark will take the first one it finds, so the version may not be expected. Possible values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same revision, NEVER: do not allow multiple jars at all.|SAME_REVISION|Startup spark.rapids.sql.castDecimalToFloat.enabled|Casting from decimal to floating point types on the GPU returns results that have tiny difference compared to results returned from CPU.|true|Runtime spark.rapids.sql.castFloatToDecimal.enabled|Casting from floating point types to decimal on the GPU returns results that have tiny difference compared to results returned from CPU.|true|Runtime spark.rapids.sql.castFloatToIntegralTypes.enabled|Casting from floating point types to integral types on the GPU supports a slightly different range of values when using Spark 3.1.0 or later. Refer to the CAST documentation for more details.|true|Runtime diff --git a/iceberg-common/pom.xml b/iceberg-common/pom.xml index 57520e19d74..5d0624758bf 100644 --- a/iceberg-common/pom.xml +++ b/iceberg-common/pom.xml @@ -42,7 +42,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index 874a9e74b34..c5df79cd5e9 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -17,7 +17,7 @@ set -ex -## MVN_OPT : maven options environment, e.g. MVN_OPT='-Dspark-rapids-jni.version=xxx' to specify spark-rapids-jni dependency's version. +## MVN_OPT : maven options environment, e.g. MVN_OPT='-Dcudf-spark-jni.version=xxx' to specify cudf-spark-jni dependency's version. MVN_SETTINGS=${MVN_SETTINGS:-"jenkins/settings.xml"} export MVN="mvn -s $MVN_SETTINGS -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 ${MVN_OPT} -Psource-javadoc" diff --git a/pom.xml b/pom.xml index 4b776c3a0a1..746fbf8ab3b 100644 --- a/pom.xml +++ b/pom.xml @@ -1061,7 +1061,8 @@ spark${buildver} cuda12 ${cuda.version} - 26.10.0-SNAPSHOT + 26.10.0-SNAPSHOT + cudf-spark-jni 26.10.0-SNAPSHOT cudf-spark-private_${scala.binary.version} 2.12 @@ -1275,8 +1276,8 @@ com.nvidia - spark-rapids-jni - ${spark-rapids-jni.version} + ${cudf-spark-jni.artifactId} + ${cudf-spark-jni.version} ${jni.classifier} @@ -1572,7 +1573,7 @@ This will force full Scala code rebuild in downstream modules. failonerror="false"> - + diff --git a/scala2.13/aggregator/pom.xml b/scala2.13/aggregator/pom.xml index ced3cd10363..746e83ad015 100644 --- a/scala2.13/aggregator/pom.xml +++ b/scala2.13/aggregator/pom.xml @@ -166,8 +166,8 @@ org.slf4j:* - - com.nvidia:spark-rapids-jni:* + + com.nvidia:${cudf-spark-jni.artifactId}:* org.openucx:jucx:* diff --git a/scala2.13/api_validation/pom.xml b/scala2.13/api_validation/pom.xml index 18dd39f2b1b..72c5e50eb6a 100644 --- a/scala2.13/api_validation/pom.xml +++ b/scala2.13/api_validation/pom.xml @@ -43,7 +43,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} provided diff --git a/scala2.13/dist/pom.xml b/scala2.13/dist/pom.xml index c9fe0648384..cc8546c735d 100644 --- a/scala2.13/dist/pom.xml +++ b/scala2.13/dist/pom.xml @@ -445,7 +445,7 @@ self.log("... OK") com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} META-INF/** ${project.build.directory}/jni-deps diff --git a/scala2.13/iceberg-common/pom.xml b/scala2.13/iceberg-common/pom.xml index 0390c3fc5d7..a972444c716 100644 --- a/scala2.13/iceberg-common/pom.xml +++ b/scala2.13/iceberg-common/pom.xml @@ -42,7 +42,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index bd11ffcbcb0..a59788cc7ef 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -1061,7 +1061,8 @@ spark${buildver} cuda12 ${cuda.version} - 26.10.0-SNAPSHOT + 26.10.0-SNAPSHOT + cudf-spark-jni 26.10.0-SNAPSHOT cudf-spark-private_${scala.binary.version} 2.13 @@ -1275,8 +1276,8 @@ com.nvidia - spark-rapids-jni - ${spark-rapids-jni.version} + ${cudf-spark-jni.artifactId} + ${cudf-spark-jni.version} ${jni.classifier} @@ -1572,7 +1573,7 @@ This will force full Scala code rebuild in downstream modules. failonerror="false"> - + diff --git a/scala2.13/shuffle-plugin/pom.xml b/scala2.13/shuffle-plugin/pom.xml index 3aac09f8218..dcaddc2ca6f 100644 --- a/scala2.13/shuffle-plugin/pom.xml +++ b/scala2.13/shuffle-plugin/pom.xml @@ -39,7 +39,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/sql-plugin-api/pom.xml b/scala2.13/sql-plugin-api/pom.xml index 96448cb7f2f..40310173036 100644 --- a/scala2.13/sql-plugin-api/pom.xml +++ b/scala2.13/sql-plugin-api/pom.xml @@ -38,7 +38,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} @@ -132,4 +132,4 @@ - \ No newline at end of file + diff --git a/scala2.13/sql-plugin-columnar/pom.xml b/scala2.13/sql-plugin-columnar/pom.xml index 65b2b08015e..631f05f66dd 100644 --- a/scala2.13/sql-plugin-columnar/pom.xml +++ b/scala2.13/sql-plugin-columnar/pom.xml @@ -41,7 +41,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/sql-plugin-fileio/pom.xml b/scala2.13/sql-plugin-fileio/pom.xml index 11f72e491ab..de096ad80a4 100644 --- a/scala2.13/sql-plugin-fileio/pom.xml +++ b/scala2.13/sql-plugin-fileio/pom.xml @@ -41,7 +41,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/sql-plugin/pom.xml b/scala2.13/sql-plugin/pom.xml index 07b24738358..1fc2d30d773 100644 --- a/scala2.13/sql-plugin/pom.xml +++ b/scala2.13/sql-plugin/pom.xml @@ -45,7 +45,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/tests/pom.xml b/scala2.13/tests/pom.xml index 92e28b94788..e19d25c80dd 100644 --- a/scala2.13/tests/pom.xml +++ b/scala2.13/tests/pom.xml @@ -69,7 +69,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/scala2.13/udf-compiler/pom.xml b/scala2.13/udf-compiler/pom.xml index 73a3fad9b9b..1b96ece0598 100644 --- a/scala2.13/udf-compiler/pom.xml +++ b/scala2.13/udf-compiler/pom.xml @@ -39,7 +39,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/shuffle-plugin/pom.xml b/shuffle-plugin/pom.xml index 5e4b9051ddc..eef2e93f9cb 100644 --- a/shuffle-plugin/pom.xml +++ b/shuffle-plugin/pom.xml @@ -39,7 +39,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/sql-plugin-api/pom.xml b/sql-plugin-api/pom.xml index 04d1ec1dca5..dabce7f1b03 100644 --- a/sql-plugin-api/pom.xml +++ b/sql-plugin-api/pom.xml @@ -38,7 +38,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} @@ -132,4 +132,4 @@ - \ No newline at end of file + diff --git a/sql-plugin-columnar/pom.xml b/sql-plugin-columnar/pom.xml index 3ab08e912eb..3f2d8666a11 100644 --- a/sql-plugin-columnar/pom.xml +++ b/sql-plugin-columnar/pom.xml @@ -41,7 +41,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/sql-plugin-fileio/pom.xml b/sql-plugin-fileio/pom.xml index 01f9c7e3600..40028e94837 100644 --- a/sql-plugin-fileio/pom.xml +++ b/sql-plugin-fileio/pom.xml @@ -41,7 +41,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml index deb838686ee..7fc717b9bb5 100644 --- a/sql-plugin/pom.xml +++ b/sql-plugin/pom.xml @@ -45,7 +45,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala index ee616a43b28..658edd804cd 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala @@ -82,7 +82,7 @@ case class ColumnarOverrideRules(sparkSession: SparkSession) extends ColumnarRul object RapidsPluginUtils extends Logging { val CUDF_PROPS_FILENAME = "cudf-java-version-info.properties" - val JNI_PROPS_FILENAME = "spark-rapids-jni-version-info.properties" + val JNI_PROPS_FILENAME = "cudf-spark-jni-version-info.properties" val PLUGIN_PROPS_FILENAME = "rapids4spark-version-info.properties" private val PRIVATE_PROPS_FILENAME = "cudf-spark-private-version-info.properties" @@ -210,7 +210,7 @@ object RapidsPluginUtils extends Logging { def detectMultipleJars(conf: RapidsConf): Unit = { detectMultipleJar(PLUGIN_PROPS_FILENAME, "rapids-4-spark", conf) - detectMultipleJar(JNI_PROPS_FILENAME, "spark-rapids-jni", conf) + detectMultipleJar(JNI_PROPS_FILENAME, "cudf-spark-jni", conf) detectMultipleJar(CUDF_PROPS_FILENAME, "cudf", conf) } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index ff2533e7d4c..a53f29b2990 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -2569,8 +2569,8 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. val ALLOW_MULTIPLE_JARS = conf("spark.rapids.sql.allowMultipleJars") .startupOnly() - .doc("Allow multiple rapids-4-spark, spark-rapids-jni, and cudf jars on the classpath. " + - "Spark will take the first one it finds, so the version may not be expected. Possisble " + + .doc("Allow multiple rapids-4-spark, cudf-spark-jni, and cudf jars on the classpath. " + + "Spark will take the first one it finds, so the version may not be expected. Possible " + "values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same " + "revision, NEVER: do not allow multiple jars at all.") .stringConf diff --git a/tests/pom.xml b/tests/pom.xml index d26afb68404..4c6e04a60e1 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -69,7 +69,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier} diff --git a/udf-compiler/pom.xml b/udf-compiler/pom.xml index 5e64c60363e..fa891354220 100644 --- a/udf-compiler/pom.xml +++ b/udf-compiler/pom.xml @@ -39,7 +39,7 @@ com.nvidia - spark-rapids-jni + ${cudf-spark-jni.artifactId} ${jni.classifier}